diff --git a/cld/clusters/v3.2.0/README.md b/cld/clusters/v3.2.0/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8f584574bde67ba934df695fcc32e7d543a586cb --- /dev/null +++ b/cld/clusters/v3.2.0/README.md @@ -0,0 +1,47 @@ +# CLD clusters: ParticleFlow v3.2.0 + +Attention and HEPTv2 checkpoints from the 28 October 2026 backbone campaign. +Both complete 50,000 optimizer steps with seed 12345 and global batch 128. +Training code: [`6b80b983477321cf708e5043fb1d2ec15dd398bc`](https://github.com/jpata/particleflow/commit/6b80b983477321cf708e5043fb1d2ec15dd398bc). +The v3.2.0 directory is a release label; this upload does not create a code tag. + +| Backbone | Original run | Parameters | Validation F1 | +| --- | --- | ---: | ---: | +| attention | [cld_pf_attention_seed12345_20261028_111201_967206](cld_pf_attention_seed12345_20261028_111201_967206/) | 3,114,258 | 0.8352 | +| heptv2 | [cld_pf_heptv2_seed12345_20261028_111202_024020](cld_pf_heptv2_seed12345_20261028_111202_024020/) | 8,639,954 | 0.8470 | + +## Files and checkpoint loading + +Each run preserves `checkpoints/checkpoint-50000.pth`, configs, the resolved +`particleflow_spec.yaml`, `model_kwargs.pkl`, scenario manifest, every training +log, complete available histories, separate `runs/train` and `runs/valid` +TensorBoard event streams, and `plots_step_50000`. The checkpoint contains +`model_state_dict`, optimizer state and `extra_state` with step 50000 and resume +state. Use the accompanying configuration with compatible ParticleFlow code. +The original full checkpoint bytes are retained. No best-checkpoint selection +or inference conversion is implied by the filename. + +[release.json](release.json) records dataset versions, model dimensions, +checkpoint hashes and metadata validation. [SHA256SUMS](SHA256SUMS) covers the +release contents except the checksum file itself. + +## Interpretation + +PF/clusters refers to the track/cluster-input MLPF model, not Pandora output. +Hit models reconstruct particle sets from raw hits. HEPTv2 and attention have +different parameter counts; hits and PF also differ in decoder, objectives and +target populations. All measurements are single-seed observations. + +Particle F1 uses capped matching cost and requires deltaR < 0.1 and relative +pT error < 0.5, without requiring PID agreement. PF uses TFDS 3.3.0 and hits +3.2.2. Validation uses 1530 PF entries or 1536 padded hit entries. Detailed +momentum uses a 512-entry subset with different selection across representations. +CLD hit HEPTv2 is missing its 30k history JSON; the final 50k result is present. + +[Comparison summaries](comparison_metrics/summary.json), +[metric definitions](comparison_metrics/provenance.json), and the archived +[study methodology](comparison_metrics/study_methodology.md) explain sampling +and timing caveats. Relative artifact links in that archived methodology refer +to the original local study; use the release links here for uploaded files. +Predictions, batch parquet dumps and dataset caches are not part of this release. +Training timing does not measure inference or upstream tracking/clustering cost. diff --git a/cld/clusters/v3.2.0/SHA256SUMS b/cld/clusters/v3.2.0/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..ae2703ca1150e623cf30056d5453a6ad07b073b2 --- /dev/null +++ b/cld/clusters/v3.2.0/SHA256SUMS @@ -0,0 +1,197 @@ +69f120e6930be31d91d381038086df2456e45e8b4542eae4d341fa43112c4f33 README.md +38f74c605f67cfc429a5394f30856f3d9e65bf40316708ede5eb0aa7597d5000 cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-50000.pth +9885a0dd835806794edbaebb04ad24f92f122fbe6069bf699d5f02a9a16e3f3b cld_pf_attention_seed12345_20261028_111201_967206/history/step_1000.json +c8e7114da2dff1738f4c6360ac0a7b3045204a22542395a4fc53015d8e9ce028 cld_pf_attention_seed12345_20261028_111201_967206/history/step_10000.json +38f9a6663a5f14faf7ace9330d45a1b7a272f5bdce32689b8f09def60f418bd6 cld_pf_attention_seed12345_20261028_111201_967206/history/step_11000.json +2f43932525b97a337657e3133b6a8c6f36e6eb507cb2a4eee6205d3eda83271f cld_pf_attention_seed12345_20261028_111201_967206/history/step_12000.json +8a64f7bfb214426a641b7992eea82061c95a65a8d6d2d61d96800bf34fa7e32d cld_pf_attention_seed12345_20261028_111201_967206/history/step_13000.json +c9f3b374f114186992f9e1390ac5d7592c9ae94694c4de3574e0a2dbed95ba60 cld_pf_attention_seed12345_20261028_111201_967206/history/step_14000.json +02daa2b5e18d7205fd74b395c734b99e72cfc609d03770dbb480aed6d4bc375f cld_pf_attention_seed12345_20261028_111201_967206/history/step_15000.json +31f1b535bb54169e77df18dc8985cc2667356322e41ba70f5fc8f202aa1a8727 cld_pf_attention_seed12345_20261028_111201_967206/history/step_16000.json +5a7a7f72f7b2218ffd85d2bb521ba4fdfed100f086bee820f24a1e8594d43d27 cld_pf_attention_seed12345_20261028_111201_967206/history/step_17000.json +8f0938dad79f8a899837b017b4cf7b58ee96e8354af7e130714a15d4b5ac186c cld_pf_attention_seed12345_20261028_111201_967206/history/step_18000.json +14c263f204bb2e98657a9c5633183316fead4b66e37c16e02091f3de63426042 cld_pf_attention_seed12345_20261028_111201_967206/history/step_19000.json +aa576f7b68c073f552a63f196325a3e6f828f1a1fab31b87878a4e675f021940 cld_pf_attention_seed12345_20261028_111201_967206/history/step_2000.json +ce223e89fe9f0829cf090266f0f98f8f2daa99842a3fadd64ef4d8d0d0af48d3 cld_pf_attention_seed12345_20261028_111201_967206/history/step_20000.json +5036932996d020d81d69ad617320db66b91fff39b388db0991a53ef6a8975efd cld_pf_attention_seed12345_20261028_111201_967206/history/step_21000.json +3fd5cf5d934a82ed03c3d3fe6f8d0cebb172a3c9baa3ec86bb953bd69d298b6d cld_pf_attention_seed12345_20261028_111201_967206/history/step_22000.json +edb4622d8f800cbc6086f795f120823207de2ed529da7603aa22218397c25eb6 cld_pf_attention_seed12345_20261028_111201_967206/history/step_23000.json +b1a9563b3081edcadd3fa52a23b1389408b864e482dc3e9a50c84d81b5b3ae33 cld_pf_attention_seed12345_20261028_111201_967206/history/step_24000.json +313df51eca1b1ff43a5e30c13c01025c17f172598ab7ea5486f5ca390f0b2b21 cld_pf_attention_seed12345_20261028_111201_967206/history/step_25000.json +d9258c5cc24d9280f898e2f17bc08e1eaffdd190ca1f39ccae0f31f9ab58086f cld_pf_attention_seed12345_20261028_111201_967206/history/step_26000.json +ae3413c4878aabddfb97f8ba2d1c415df11a4ffb9d236b574d03250ea1059216 cld_pf_attention_seed12345_20261028_111201_967206/history/step_27000.json +d1ad58377a4616d5345ec76720e1682dd895d4a25bf24d846929425f73161f17 cld_pf_attention_seed12345_20261028_111201_967206/history/step_28000.json +9bcfe5fbf9aa65e7f0e98e132be0d5ed91115aa27ebcd1bdadad1b525ca3bfcc cld_pf_attention_seed12345_20261028_111201_967206/history/step_29000.json +89531cc71eabefce3c78088062ba8ef95654765a575d7ca084a50208f7b89de2 cld_pf_attention_seed12345_20261028_111201_967206/history/step_3000.json +48c059f5faea5c352cc47ce7a78dbcfa2b602e72066d35795bf7fc91d62bfb83 cld_pf_attention_seed12345_20261028_111201_967206/history/step_30000.json +97d1a8b4cf07af4d7ea802a66b28509e1ee46915def2754b7f38b45f3c1cb8b4 cld_pf_attention_seed12345_20261028_111201_967206/history/step_31000.json +54f0b6b086675c0184526855864bf637c1ce9dd517931b780a4cfdc87e9dd2ef cld_pf_attention_seed12345_20261028_111201_967206/history/step_32000.json +e4e19d7369a2e192eaec22613922b7cd39f55b214ff07b613b132603880382d9 cld_pf_attention_seed12345_20261028_111201_967206/history/step_33000.json +ccc6da6fc9ac8d11a1dd99075582e10f6739437e36569cf15a045eb60f7d5769 cld_pf_attention_seed12345_20261028_111201_967206/history/step_34000.json +e3d0dec1fc775b07d2ac21ebfc753b39c9ffd90be0e5c592346c504fda7593aa cld_pf_attention_seed12345_20261028_111201_967206/history/step_35000.json +028760115a8683927e7cc042dfb5e3ccf6b132cdbc4b0a42677a4c2362c80bdd cld_pf_attention_seed12345_20261028_111201_967206/history/step_36000.json +b7f285b30e127d935530365f88ec67ee170774756dfea926f0328649f9da395b cld_pf_attention_seed12345_20261028_111201_967206/history/step_37000.json +a26623eadbc53e14f21242ff379a31a69254d26456e35177545006a75a230b0f cld_pf_attention_seed12345_20261028_111201_967206/history/step_38000.json +6873ee78660f8759c9ff74c82c7dbfe75f008e8d241655d14daf91d346f49131 cld_pf_attention_seed12345_20261028_111201_967206/history/step_39000.json +396bcb173865eb338486860cc0f697ad468055d214858201c859e18eaa3297c7 cld_pf_attention_seed12345_20261028_111201_967206/history/step_4000.json +2999dca6193d8d2c41b36df1e168ddcae2ce9f13a911162b7f29bdab510da33c cld_pf_attention_seed12345_20261028_111201_967206/history/step_40000.json +850a8ffd3f38fe98dd52ad3ecb247c80cf2bbb1d9f0330a6e69b2a8ccf3020b6 cld_pf_attention_seed12345_20261028_111201_967206/history/step_41000.json +bac420ab443f8be71627dde361fd51b461519404f7e1f4875b78929fb09027ce cld_pf_attention_seed12345_20261028_111201_967206/history/step_42000.json +cedecff4a73693d529b7fc2a23c2266a067be9faa53ca654ee898babe5bf92a1 cld_pf_attention_seed12345_20261028_111201_967206/history/step_43000.json +eee07bff668b136f842bbb28b8043b84f352e82f9067fe3a5cdf654d85683b09 cld_pf_attention_seed12345_20261028_111201_967206/history/step_44000.json +4bf8e0929850aa33152050ac76e68d9820afb1d2847ffa89c865fc22c9652204 cld_pf_attention_seed12345_20261028_111201_967206/history/step_45000.json +f8a42b72755475bf4aaff3542a662b598adcda7873705e3378faef2c5b4123cd cld_pf_attention_seed12345_20261028_111201_967206/history/step_46000.json +7d63374760fda19de7847020cf539354c6f3a958d50fc99b1a7c4ac23838e486 cld_pf_attention_seed12345_20261028_111201_967206/history/step_47000.json +4841dcdb67e848e5d0f85b8a8d30189adbeec582706b8babefc688dcafa6098c cld_pf_attention_seed12345_20261028_111201_967206/history/step_48000.json +9b0583fcafd23310d604ff10ef5dd0577a799d6e13f82593cbe2a58e8c70dd24 cld_pf_attention_seed12345_20261028_111201_967206/history/step_49000.json +a1f363df2bdde57fa61233aedb546df00038c8bcb60425f0d417b6168396722f cld_pf_attention_seed12345_20261028_111201_967206/history/step_5000.json +114c06e44432eac31a6bd80c9ce89777bbc915ea6158a67fbdaff71cf6943597 cld_pf_attention_seed12345_20261028_111201_967206/history/step_50000.json +e3ddbd79aa431b92729143a7b5aa982c596c3209347c9b5548bafc17b9e3bcc3 cld_pf_attention_seed12345_20261028_111201_967206/history/step_6000.json +1647cd956ec5d90ea9d7d8a3730e75c7753c578eec6052853cc97003c284bca8 cld_pf_attention_seed12345_20261028_111201_967206/history/step_7000.json +266453b9940a5c482e549473d482d83488a6986d53de180363b67fb6856e65c0 cld_pf_attention_seed12345_20261028_111201_967206/history/step_8000.json +0ad7fb1126a58b132fd9ea0e248826d2f45266e309253d4c3d6916a2cc4f82c0 cld_pf_attention_seed12345_20261028_111201_967206/history/step_9000.json +fec6d66ca4a9b85a134c8d5362b7ea0da2e1735ff35b0031722f84310455b894 cld_pf_attention_seed12345_20261028_111201_967206/hyperparameters.json +f2477072065a5878ead61a8d52902e829e79b4bddf244254c1319529f621c25d cld_pf_attention_seed12345_20261028_111201_967206/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 cld_pf_attention_seed12345_20261028_111201_967206/particleflow_spec.yaml +4bc37bcbdfe463c3d08b60a023e4177a4048d54bd332fb8bc7e81f8f13243a94 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf +5335096c0b18140d27908fc20b4d42de35210cbb84a80a7bbf56281657f58622 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.png +e17bfe59a7040c7a56cf42d0487e92ca8791719c2cd24fe69621652371955a47 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf +0a3e45aa1b91f8fc4aa809181ee0c0e9f7dbf3a7ed8704d80021553f01fdde83 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.png +7acdb97ceec338602ac81eb009d0d6921c0c9bb0b0d243b6da85c07d1fe1371b cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf +0c78c34b3c1aa9fb0006bc704edbf5dac87b6adf84cb45303beeffb30b72992f cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png +49af7e4fcc579466ae5028810a325026702999f67d80815a179b0fcd0dbacd29 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.pdf +259ce699170eacec56c9ab20241d180e66e945d17f10b1a2c35df5a2c244b1c4 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.png +53302cd1c8dcf3d4c8a576aeda8ea7037c645bd7b20e34ebab011cc015a8f819 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf +5ae35f211df360005e82479b66e3264cbd9310742af590f068cb777eb5e5031b cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png +d976bfd1780d4c4500e5a7059b0127b82e63d15488a9d0ad4cea9d075a6382ac cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.pdf +e6f42a756fdb17e983c63cdeaebfa5cc4a606ada2fddf3ca9dc10e143af62977 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.png +3065a2d37c466f23702c2906071418fde4785c4c4926bc2329a0ddffac5874f7 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf +77a83e53feaafdbcb12d606c3aad7ae182d41f0f5c237c7621719b8cc4e0b2c0 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png +0fe752de3586b90e69528651b0372aae747782d24966ca7f1b8be4a63eb999d3 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf +87c1b85f81b0907f55e8a957082a64ecacb4957663abd3bbba900386ff61c920 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png +af32a3f73e5f6a1c1a39324f74e56352f54d628320d6e86ea38c69a1618ce8f7 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf +be81da04e25bd655ed18e7591f195870ab4f6122b0768b4713fe96c2eff9c826 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png +2a20f89036c0204b5d943c1115d0254c31569ff9a299e18d09e12ff32463c239 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf +108353f00d01b3b3835e3adc45eb5ed0b919bfd52cd229e73e3e2cf0132361e8 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.png +fbafc6c160a33a644bd91e02af106b6b28a30b06e9516e5e2949897f8a4b7e2d cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf +d3cc2f95474a0c7092cff4e0056f8e31fd98582ee5517449bf37e3768b1ad142 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png +82a5017fe5734c536a924650a00c8914c7190befa690b9ffa46406a28b5e0e9a cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf +c55b3691cc379008dfe28f07906ecc3e4183bdd7981f81f2a196caa0e9c1048f cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.png +692f3503aead60d0ac660b6c28bc4f318ce152219c2eeba12328807b5a06a43a cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf +3472b85765e7cf77f28f83445e95ba9d548ceba3556f3a6c3f2ab38641a92efc cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png +69240a5d12da88094d9ec0e97cb74eb28b2e06fd90e5fe7ba8bd989df06af32f cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf +98e6afa3d9d712264c45c07e927e33242a346b9f4342f0ff5549820555a50879 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png +34876b7471ad600eebb40693523ca8150cec32c72817fa97911d63a3189cdd86 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf +1e4eb25cd2df3472bbc6903b61108b191e3fcf72ac8c2878b225cbb1da6f9cdd cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png +7367caee71efa0ba2f559ad2288c0ded56c3f89ce3b01a1d8d77a65ac724fd39 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf +321d767ca36ad5b5e7e08eb988fb841dcbad7875f9e8bbc5a2c36153e01a375d cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png +89b166c11ec6f81732f5338a042c9538991b9dc8590fe9b351a96f55ec213b8f cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf +8b340b689b8b253e5d18813f26ce62e9a4f4195730bc5b5f88006b499b38df17 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png +a6d97cc480bd0d3074b68bb8bea3852f1a90e77280b03591f117094a5011149a cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf +5ca55d519abaeb1b6cfc805fa5ca4373abfbd8f42b994f5d1f0a822f588666d2 cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png +ceef311fd91157f1393962913b350425e25bac550671f943ceac7b531af759ec cld_pf_attention_seed12345_20261028_111201_967206/runs/train/events.out.tfevents.1793178753.nid005546.24715.0 +fc73b83d71bac222dc3d0965e2664b5038abd24ddd2a47950ca9548f1d8ee714 cld_pf_attention_seed12345_20261028_111201_967206/runs/valid/events.out.tfevents.1793178753.nid005546.24715.1 +0ec507f99c409beb914ad09109ea78ed4d062eb5d2ecd25759f4daf2701cbe94 cld_pf_attention_seed12345_20261028_111201_967206/scenario-manifest.json +bae37a018fcffdcd44e312de4615f357d890671c1ff98f4dde5ca80c576f63e7 cld_pf_attention_seed12345_20261028_111201_967206/train-config.yaml +3a10ae81d9296813b823302603baa63897568272ffdd6b1caf5b7b9e890536e0 cld_pf_attention_seed12345_20261028_111201_967206/train.log +ee5e75b0bcf82c690b3fd15845a8db3d94eafd7c2e5dac74ca30fe9fb3937a5d cld_pf_attention_seed12345_20261028_111201_967206/train.log.0 +01e13a81252b32cd0ba43135e00c12b229aee6e8a9951af527a956d65e947b2a cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-50000.pth +97890799b685f8b12d3681c270706cd3db001705a83e25500bc1e00d5d721c11 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_1000.json +7c8c0b690cbf56cd68c152d2fb2fafd05a2b3d1f34354edb6739047fb0328548 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_10000.json +2ac00105a049588ded5ebe7f92735595cd524f5097ad94470a71b9be8afb4542 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_11000.json +68e88cc746f1dcca8637978bd5820bd078b87f3c1386b27f4b1771812068b6ca cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_12000.json +51ef9f9a4a4ba895c676ef34b2ba7634458b5d94ef524c95dd997dfc91c612f0 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_13000.json +de26e194bdcd3682aedc18da4f237a20509a3af47169cfbb141d64f8dbd27713 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_14000.json +a6e628e49182ed00f508287e9a93840ecab415a7c20bc597325941068dc68d65 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_15000.json +609c653e7f19173d8a843236e0a55868c540fddde50140cccbc55dbf6f47ba09 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_16000.json +8222dce7cd962e412c1af582e19075c9e880ec553a86b3ab0d58cb724fb78f28 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_17000.json +3f49f266abe0b46f77d62d0c39bd1741e85a82430481be1603d76b9eceb7df4b cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_18000.json +725557c682bcfa52289269de419699ae0a09d0e57b1d79fddf5a591ab969cb21 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_19000.json +503c19ad56dfdf69616cba00b98c3ab6be6fe7fc3f2fcf0e1eccac8245056f59 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_2000.json +cb0ecf51e4ebe2078a6e96793f2723fbba9c501bfd9d408588e5e86b9bcbd2f8 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_20000.json +1c0413994a4d48ca39ab41f2721407b70abb14aec4748104cccfdf9fe97ffb65 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_21000.json +db86054bf18e1783df946150981ad4507aa9cfc56aab694a1aedb3f0b3da4048 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_22000.json +21735619aafdc434ce17c670d03270462102b0dd98e3e502e76e0606fba7a1b5 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_23000.json +234352728f8e2f3d26a7fbca6b12a66c46b2b90404ea9ef21d77ee86d0486910 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_24000.json +87f6e4345e135f9b9c3c4b2ebfb6ae00c37985aed364cb3d6bdc763bb0ff9b7c cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_25000.json +c360eb0694077c1f67e0b57e7a3ed8bfbcdcfc3eff36453fcf72f145e56ccac9 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_26000.json +f9320b66f35bd16f22676386bfe6c1b68ccc050ca8f9592af0e004ce07213a9b cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_27000.json +c84f29a24989fcb55e4b79868e89322e1dda99ae80c52ce0c3fe816dbfc55a54 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_28000.json +38a94499989f9d8760c3378257a88b1acae7901b7119258704a947a3545bcc04 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_29000.json +9042ac1074ada262075bef2920debf78ddc19391f5e3f7e6397f073c303934fa cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_3000.json +badb3c0f575e6658010a33bbb57dc1fbe9b13067430abd9c6e6dcf67da35541b cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_30000.json +2b856b5c1ba5ddff0a93e7df3b57120794fb0d4b7a050f244542dcc450e9e093 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_31000.json +37ec9cc2dd2480df8e0ed07da87e96d093539e82826d5678163effddc738a375 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_32000.json +5230628704df2c62da42efbe04f7d75000e5c105d2977dd2027e3dbcbed1c722 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_33000.json +23ccbc12ef7f40e9d02b8df2729ecd5f8ecf415a9300b9d9d0caa84a3ea88e74 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_34000.json +1dbcde27bc9376c52d01b28073e5ba9bc328003c5ebd610a3cbd92335b75ac8b cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_35000.json +2599ebc1662035eab147f3152551651ebfa87833243670b62baf6354c009f1de cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_36000.json +e35e269806282cd0dcee656146b77187e82f2dbca8bc32acbe4d8e51524de1db cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_37000.json +4ad3c922b3f244f27ebc9ecc4662e2a38c97bc704903c3f4e3e1779fbeb691c0 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_38000.json +8455a69b2866771237f054face7e765c2266e98028436e9320c56b77e15e0849 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_39000.json +87d5e0406a1d1918ab76126beacdf276fc5a6199c9cf509a368f47cf4e388b2f cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_4000.json +f842029878cb486a78d5bb9643a76438c70960444a72eaf7de714a0422c91e9b cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_40000.json +32df95f7aa580064b1687191acfbe61d411e01b1be9370a61a714a7bac5de9d8 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_41000.json +a8405214f44408537b5376f5033f202d5322d24576e15704d1a00fc2a98bdb30 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_42000.json +dbd2fc200d85aab997a2330d577ae288909193b03e739cc5f81eb678f7e51157 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_43000.json +2eec5383cb462308419013d9c7689728b9a66874ab1be22b7e0b5f1a633db5b7 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_44000.json +a61258d7ab4609b6a4353ce2c17f8e3d1cc8a3259601ac04bfed5981d7d4d952 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_45000.json +6cf2704be938b65f4bddb4eab321271a1321448d62aa792ea85c004b6d606924 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_46000.json +d6c4d2c65b3ee52612ad381b99fa70899a63ef252e03efce29690a3eff2db8a3 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_47000.json +c9b62ed30e5741c01c2e60eb9cae0992cf7e2192cda769b2afb29c140aa13f53 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_48000.json +787798f97d74396b9b60921007d44ef788b0689602ab24f1f5464c436a847786 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_49000.json +63bb974ab348d8fbbe75627ff05a431ffdf79a6d362369c01300db16e1e23f8c cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_5000.json +4fe11bdbeaa7c131d098003c4d601686fe5c77f022501384209f729b96157814 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_50000.json +d3b681b05296b147c4034f0bf0de1857e2acdbdf196c53e1ec5e7250a3964a97 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_6000.json +7f1fb2819b2d569d78471b647f25f0b1e74677885d2a8eadfb174c3935906cc2 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_7000.json +fec58271080e440fb266a417fd42a9e3902fbad7bb1aa41867b842ba1e37d4b4 cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_8000.json +b44d81e4d27ac5aff986f10db740949942572433ade6ac56eaa9ed8ff931620b cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_9000.json +d66aa5d3c4412828771b338d6dbc206b7d21ed55bf3e604c46b00f906899a8da cld_pf_heptv2_seed12345_20261028_111202_024020/hyperparameters.json +9630a465891b2b9e7774c6212493d87ae56afa1238f059daff58af24aac811fc cld_pf_heptv2_seed12345_20261028_111202_024020/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 cld_pf_heptv2_seed12345_20261028_111202_024020/particleflow_spec.yaml +72a3920592fb6d522be65272a304c4f32ee8aa84301f8efdcfde46cd407e6210 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf +bfd01491e0797d8819f637874e555a46e32a68c8716dd2bdd3d389ede9e32420 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.png +39b1fcfdc1a1a4875c763dc73ee1a75be9d0bf2e567b160e831b65459416764f cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf +79d9b272c4176c2ebdd6d398a619cf64307b20b938f7e746e4b7712544e6cfbf cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.png +1cabe1616194b3e29335e7fc1ab0335456b41ed607d5a863952c6b3104866154 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf +e726ca638e6ab8d867721ac08cc28624cab77b9c443592ecad41b9f923a209d6 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png +2a992bf8b40acb03cc8f0c3fd9229fa81caab42e7445fb82d4a194b17ad9fb01 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.pdf +5d19a9fa56ebf645906cf44a1b29ee891955c7be4e09dbc193dad497d5cfe251 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.png +48561f95fe2adad57e533408589ea4d807bbd44f42d96b86de1d298f994c57f0 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf +9427c57f63ba81d474fa5542a25702b4406e447cbe9cc48d58d75913795f1415 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png +4612488ef83f482889d46fb13739082ac7bbbc9cb3416dc5c9d90f8c72998f1d cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.pdf +e6f42a756fdb17e983c63cdeaebfa5cc4a606ada2fddf3ca9dc10e143af62977 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.png +a126c518320af7554043e3e560a5853878a9c5f414dad2279d955ee0864002eb cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf +56c681946c280c3f7be1cb752df04cad00fe8c43ce60d7fa02f8e5ff2ccda2e9 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png +1ec88c132d5b350aed8e22d304f07f734bd5db3839bbeba36acd1979a70d4852 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf +0d21f42cbaac77fdc73ef4c8363a5aa08c23032a4a74f0e8322a683d1723ed6f cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png +3e1b14cdd571be2700b72ead5d529b9c1904f95f800fdf233b6817fac54eab62 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf +7434bbeda60f8328df755bd073bb2aa7154339888488b9275371814b4d036dc3 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png +9b71fe6ddb511c2d8ff43b16e1df6ece249a856db1d560e9bf96ed8b1a21742e cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf +43ec5b02f75270de4bede77a97d8c3fd1b0d8bf092781d6d595824673ef35498 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.png +14314239b372ad3a9804ef4d458a453b073a9ca4e9c419cb65484a0e65311cf0 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf +ddd3ff970e1446440b0085c03fe9eaa00336078dea3020b57af7831c87451919 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png +8d67b192f773a6bf9801b3455333c28e012336203cdc8de6e53aeeefca387238 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf +c55b3691cc379008dfe28f07906ecc3e4183bdd7981f81f2a196caa0e9c1048f cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.png +d83783bafbfd7d5bacaf342cb90bc8afeabc830ec2eecd2f3325d5edae1881e6 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf +4b4f32d92b819ae3d980e93878ab5ab38c407dc10499bd5b39c2fa169c7df6e1 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png +0cb8d3e4dcc87b13c3e8dca201535a0da1b998b53deec58dc89a7cd960f67cc4 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf +8936f4d62f7b1f0f667325cfea984b49f6097adccf513340a6311791129369d2 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png +289464ea2e2639406fa4a3a8f0f8ca686ceffdff4760037631672d22b31738f5 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf +f040c1db3d35356aeb1397c14251fc3d31d4672229bf9ecf01c5bba152b4837c cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png +c73059bd8e610f0a9927e9b173858a3e8c28cdf02377a24bd23853cf9939eadf cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf +a73292b3f168bbbd2a011ce44577bfc90f32f52e7e52a4f98debe852eed9b01a cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png +7df84bb3926ac170bcd6adff4809a9b70a237cfbfcd9cb4db9dba5add24736f0 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf +73dd049395175db38cb6d4c26f701008197773c9dcd885924b1d98144d3557b4 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png +7b5b794f6692073d4c40742cc00e561c99e3e252afd3f2be43afedb97832489f cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf +5ca55d519abaeb1b6cfc805fa5ca4373abfbd8f42b994f5d1f0a822f588666d2 cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png +05675302666a98fbaebe9d3bd4a026065398efc126c03507f98169b56df9df91 cld_pf_heptv2_seed12345_20261028_111202_024020/runs/train/events.out.tfevents.1793178753.nid005725.85735.0 +ee9f0a6e7e928c91eca10a03e0f4a3aa9eb342fea1fd8c719c54eb981bc2b084 cld_pf_heptv2_seed12345_20261028_111202_024020/runs/valid/events.out.tfevents.1793178753.nid005725.85735.1 +8cc2b2ba31a94f39c6b612be9c786c0644548a56f5f053c01fa8b644693df70c cld_pf_heptv2_seed12345_20261028_111202_024020/scenario-manifest.json +d8e75044be9a2531869c50b63b99c23a176a2efb3f7bdba96ef3afe7098e9c2d cld_pf_heptv2_seed12345_20261028_111202_024020/train-config.yaml +44e94deff6c8567834f40b48e5895d99dbba03f8caa9fc2a2718b200d63c8bb6 cld_pf_heptv2_seed12345_20261028_111202_024020/train.log +beffcaa72694be2aa6a20dfbf0c78f507247cff589ae3c5be27609c6822c30f2 cld_pf_heptv2_seed12345_20261028_111202_024020/train.log.0 +a9a813daf88031a86204a1100e12ecad2caf059954dd8256d7d98e46b2171105 comparison_metrics/provenance.json +4b00f93f03bf042116ccedcd743f4f78b0d26cc18501fca9bc294975f2e4beb1 comparison_metrics/study_methodology.md +4fadd3600235ea91968b72f348308efdc08f24f6301784d945b95a4ecc12aab2 comparison_metrics/summary.json +365558eacc557de0b91d97501f49d1f45293662927bb53392f3f02ee4b7d4df2 release.json diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-50000.pth b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..d87be057561d674bca895a189f5c715f23172c58 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38f74c605f67cfc429a5394f30856f3d9e65bf40316708ede5eb0aa7597d5000 +size 37511559 diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_1000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..131280ac0f1fc6fd1ab722fc428daba12e36eb65 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.0473664266767715, "train": {"Classification_binary": 4.568063259124756, "Classification": 0.040341880172491074, "Regression_pt": 0.2965598702430725, "Regression_eta": 0.021929839625954628, "Regression_sin_phi": 0.013420815579593182, "Regression_cos_phi": 0.014001114293932915, "Regression_energy": 0.3061119318008423, "Total": 16.113880157470703, "Total_unweighted": 5.2604287108406425}, "train_group": {}, "valid": {"Classification_binary": 4.524645487467448, "Classification": 0.03834277888139089, "Regression_pt": 0.29954081773757935, "Regression_eta": 0.01921032617489497, "Regression_sin_phi": 0.01108679547905922, "Regression_cos_phi": 0.011940553784370422, "Regression_energy": 0.307719349861145, "Total": 15.572293599446615, "Total_unweighted": 5.212486109385888, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.03837215155363083, "diagnostic/loss/cld_pf/Classification_binary": 4.505778789520264, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.011611608788371086, "diagnostic/loss/cld_pf/Regression_energy": 0.30888766050338745, "diagnostic/loss/cld_pf/Regression_eta": 0.018239058554172516, "diagnostic/loss/cld_pf/Regression_pt": 0.300955593585968, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.010197972878813744, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09217933565378189, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2766365110874176, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06506005674600601, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5654944181442261, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.058293186128139496, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2740549147129059, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08546032011508942, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5650867819786072, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.8939016461372375, "metrics/particle/class_1/pid_accuracy": 0.9776381850242615, "metrics/particle/class_2/efficiency": 0.16976188123226166, "metrics/particle/class_2/pid_accuracy": 0.10872483253479004, "metrics/particle/class_3/efficiency": 0.6372846364974976, "metrics/particle/class_3/pid_accuracy": 0.9744129180908203, "metrics/particle/class_4/efficiency": 0.8768920302391052, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.026369167491793633, "metrics/particle/count/bias_mean": 1.070588231086731, "metrics/particle/count/mae": 5.647058963775635, "metrics/particle/count/prediction_mean": 71.63594818115234, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.12341009080410004, "metrics/particle/event/energy_response_mean": 0.9701243042945862, "metrics/particle/event/met_abs_error": 15.795409202575684, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11441797763109207, "metrics/particle/event/scalar_pt_response_mean": 0.9815075397491455, "metrics/particle/event/vector_pt_closure": 0.11937812715768814, "metrics/particle/matched/delta_eta_abs_mean": 0.02241707593202591, "metrics/particle/matched/delta_phi_abs_mean": 0.017705608159303665, "metrics/particle/matched/delta_r_mean": 0.03172982856631279, "metrics/particle/matched/energy_relative_abs_mean": 0.10165435075759888, "metrics/particle/matched/pid_accuracy": 0.9431541562080383, "metrics/particle/matched/pt_relative_abs_mean": 0.09508141875267029, "metrics/particle/matching/duplicate_fraction": 0.050646424293518066, "metrics/particle/matching/efficiency": 0.727022647857666, "metrics/particle/matching/f1": 0.7215491533279419, "metrics/particle/matching/purity": 0.7161573767662048, "metrics/particle/matching/target_coverage_dr0p05": 0.6603899598121643, "metrics/particle/matching/target_coverage_dr0p10": 0.7732042670249939, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6234149932861328, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7816792726516724, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03559098020195961, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.10111768543720245, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09521600604057312, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013116014190018177, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15283776819705963, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005309988744556904, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4967309832572937, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.12295731902122498, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7740767002105713, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6968.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4878353774547577, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.7817236185073853, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.2618889808654785, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.092515230178833, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20873935520648956, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 25.386306762695312, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.026053953915834427, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3293203115463257, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0939277857542038, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.021032810211182, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9749.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6873828768730164, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8963112831115723, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 17.71554183959961, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.450525283813477, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.8575283288955688, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.1460230350494385, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 15.515432357788086, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 4.453498840332031, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 28.328237533569336, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03861308842897415, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10167668014764786, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09602022171020508, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013699159026145935, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15564070641994476, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01550387591123581, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0056402683258057, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.21033816039562225, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7110695838928223, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1161.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.1825294494628906, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.0398261547088623, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 55562.6171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 55562.37109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3995983898639679, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2772287.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.11405622214078903, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.401290416717529, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4578210413455963, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.186012268066406, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2490.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4223353862762451, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.4229881763458252, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.302895545959473, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.015425205230713, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.08497433364391327, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.20396536588668823, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.069283485412598, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.8169198036193848, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.861515045166016, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0299018956720829, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1042461171746254, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09778483211994171, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018820898607373238, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15408693253993988, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008364934474229813, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.18119043111801147, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.08436521142721176, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6712825298309326, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17932.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2853979468345642, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1709089279174805, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9275360703468323, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7631875872612, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15817490220069885, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.44163703918457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.009393871761858463, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5707321166992188, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.12222953885793686, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.595079183578491, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22355.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7843239903450012, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -5.929590702056885, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.15730094909668, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.584930419921875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.05888032913208, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.021526575088501, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 9.536271095275879, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.1890921592712402, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 16.802675247192383, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03181108087301254, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1032952070236206, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09701938927173615, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017067397013306618, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1538236439228058, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026860060170292854, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3022860288619995, "metrics/particle/momentum/all/accepted/pz_bias": 0.10029570013284683, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3285751342773438, "metrics/particle/momentum/all/accepted/samples": 26061.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.40702080726623535, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4055864810943604, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4000.511474609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4000.33984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18980169296264648, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 743767.9375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.021622246131300926, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0602264404296875, "metrics/particle/momentum/all/assigned/pz_bias": 0.072503000497818, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.705866813659668, "metrics/particle/momentum/all/assigned/samples": 34594.0, "metrics/particle/momentum/all/efficiency": 0.7289994359016418, "metrics/particle/momentum/all/event/energy_bias": -8.248889923095703, "metrics/particle/momentum/all/event/p3_error_mean": 36.76565170288086, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.42727279663086, "metrics/particle/momentum/all/event/px_bias": 2.001383066177368, "metrics/particle/momentum/all/event/py_bias": -1.0794689655303955, "metrics/particle/momentum/all/event/pz_abs_mean": 26.609033584594727, "metrics/particle/momentum/all/event/pz_bias": 9.459511756896973, "metrics/particle/momentum/all/event/pz_residual_rms": 38.14132308959961, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029337860643863678, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05862890183925629, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04808201640844345, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008723626844584942, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09668534249067307, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016364469192922115, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.24833329021930695, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.12599396705627441, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1980412006378174, "metrics/particle/momentum/class_1/accepted/samples": 15277.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1582222729921341, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4473628103733063, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6933891773223877, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.611060619354248, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08265717327594757, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.997262001037598, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010565641336143017, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5920922756195068, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0856407880783081, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.17366623878479, "metrics/particle/momentum/class_1/assigned/samples": 17131.0, "metrics/particle/momentum/class_1/efficiency": 0.8911508917808533, "metrics/particle/momentum/class_1/event/energy_bias": 12.00905990600586, "metrics/particle/momentum/class_1/event/p3_error_mean": 23.159706115722656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 14.114559173583984, "metrics/particle/momentum/class_1/event/px_bias": 0.570395827293396, "metrics/particle/momentum/class_1/event/py_bias": -1.1685935258865356, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.303611755371094, "metrics/particle/momentum/class_1/event/pz_bias": 2.9771320819854736, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.684154510498047, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.048718664795160294, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22826258838176727, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22411222755908966, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08612940460443497, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26338064670562744, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.05088062584400177, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.1218206882476807, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.29907482862472534, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.170595645904541, "metrics/particle/momentum/class_2/accepted/samples": 511.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.575024962425232, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.392804741859436, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 56766.82421875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 56766.01171875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.6011489629745483, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2802270.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.14731226861476898, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 5.4439697265625, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.43206679821014404, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 11.672837257385254, "metrics/particle/momentum/class_2/assigned/samples": 2437.0, "metrics/particle/momentum/class_2/efficiency": 0.17681661248207092, "metrics/particle/momentum/class_2/event/energy_bias": -28.632160186767578, "metrics/particle/momentum/class_2/event/p3_error_mean": 26.243986129760742, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.82357406616211, "metrics/particle/momentum/class_2/event/px_bias": -0.875420093536377, "metrics/particle/momentum/class_2/event/py_bias": 0.5388203263282776, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.910005569458008, "metrics/particle/momentum/class_2/event/pz_bias": -0.9572648406028748, "metrics/particle/momentum/class_2/event/pz_residual_rms": 29.36406707763672, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03472993150353432, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16760341823101044, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16773955523967743, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03695256635546684, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.209570050239563, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014287171652540565, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26839104294776917, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.06321481615304947, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8983709812164307, "metrics/particle/momentum/class_3/accepted/samples": 9799.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.514159083366394, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.549172878265381, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.7939178943634033, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.622314929962158, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.25132766366004944, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.868656158447266, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.014000965282320976, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8846607804298401, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.1295032650232315, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.223572254180908, "metrics/particle/momentum/class_3/assigned/samples": 14499.0, "metrics/particle/momentum/class_3/efficiency": 0.6451379060745239, "metrics/particle/momentum/class_3/event/energy_bias": 19.676189422607422, "metrics/particle/momentum/class_3/event/p3_error_mean": 20.170591354370117, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.16805648803711, "metrics/particle/momentum/class_3/event/px_bias": 2.1257519721984863, "metrics/particle/momentum/class_3/event/py_bias": -0.8105665445327759, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.185924530029297, "metrics/particle/momentum/class_3/event/pz_bias": 6.8821234703063965, "metrics/particle/momentum/class_3/event/pz_residual_rms": 22.822675704956055, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.032799310982227325, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10043744742870331, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0974399596452713, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.014573393389582634, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15214873850345612, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013071895577013493, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.829997718334198, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4148363471031189, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6717517375946045, "metrics/particle/momentum/class_4/accepted/samples": 306.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.17466196417808533, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.6841257810592651, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5325007438659668, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4214223623275757, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.10451977699995041, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.091803550720215, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8751301765441895, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.308756947517395, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.632908582687378, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8644067645072937, "metrics/particle/momentum/class_4/event/energy_bias": -6.744863033294678, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.957945823669434, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.585179805755615, "metrics/particle/momentum/class_4/event/px_bias": 0.38153544068336487, "metrics/particle/momentum/class_4/event/py_bias": 0.5870052576065063, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.008197784423828, "metrics/particle/momentum/class_4/event/pz_bias": 0.6992473602294922, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.516828536987305, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03323603421449661, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.039170071482658386, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.034854207187891006, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.012523957528173923, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07472144067287445, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0059523810632526875, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.6898497343063354, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5681061148643494, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.3780999183654785, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08677542954683304, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0588700994849205, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.057122256606817245, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.018622856587171555, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.16981974244117737, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.7566044330596924, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.6187022924423218, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.5115201473236084, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": -4.5571184158325195, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.134433746337891, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.994260311126709, "metrics/particle/momentum/class_5/event/px_bias": -0.20088006556034088, "metrics/particle/momentum/class_5/event/py_bias": -0.22613444924354553, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.5335264205932617, "metrics/particle/momentum/class_5/event/pz_bias": -0.1417267620563507, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.54051685333252, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.0052473545074463, "step_valid_time": 252.58583784103394, "step_total_time": 253.59108567237854, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9841170816220696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24087674241802648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3061, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2689, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8825073843124385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8784710878797779, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8804846103470858, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12152891212022215, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8148999015425008, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8111728193400849, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8130320890635232, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8825073843124385, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9841676330666851, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24623750468820615, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2283, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1970, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1874, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8647936786654961, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8628996933858958, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8638456478842359, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13710030661410424, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8226514486391572, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8208497590889181, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8217496163122122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8647936786654961, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9875039459258868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25746600161386635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1066, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1007, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8088012139605463, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8069644208932627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8078817733990148, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.19303557910673733, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7640364188163885, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7623012869038607, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7631678666161424, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8088012139605463} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_10000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..691ebbf75ce93664eec7c934246757b9a23855c7 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47366426676771506, "train": {"Classification_binary": 3.426697254180908, "Classification": 0.02368646115064621, "Regression_pt": 0.2070109099149704, "Regression_eta": 0.013110710307955742, "Regression_sin_phi": 0.008356700651347637, "Regression_cos_phi": 0.009828196838498116, "Regression_energy": 0.21571604907512665, "Total": 11.149839401245117, "Total_unweighted": 3.904406282119453}, "train_group": {}, "valid": {"Classification_binary": 3.3731702168782554, "Classification": 0.023753543694814045, "Regression_pt": 0.18637226025263467, "Regression_eta": 0.014641184359788895, "Regression_sin_phi": 0.008203310271104177, "Regression_cos_phi": 0.008819223692019781, "Regression_energy": 0.19280201196670532, "Total": 10.591360092163086, "Total_unweighted": 3.807761751115322, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.024051595479249954, "diagnostic/loss/cld_pf/Classification_binary": 3.365525484085083, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0084718968719244, "diagnostic/loss/cld_pf/Regression_energy": 0.1919257789850235, "diagnostic/loss/cld_pf/Regression_eta": 0.01369253545999527, "diagnostic/loss/cld_pf/Regression_pt": 0.1874358355998993, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007651303894817829, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09606815129518509, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.23202550411224365, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06894886493682861, "diagnostic/regression/cld_pf/energy_residual_rms": 0.48886701464653015, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.06440850347280502, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.22843174636363983, "diagnostic/regression/cld_pf/pt_residual_mean": 0.09157563000917435, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4852169156074524, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9107478260993958, "metrics/particle/class_1/pid_accuracy": 0.9796300530433655, "metrics/particle/class_2/efficiency": 0.36515894532203674, "metrics/particle/class_2/pid_accuracy": 0.6277691125869751, "metrics/particle/class_3/efficiency": 0.7081575393676758, "metrics/particle/class_3/pid_accuracy": 0.9757013320922852, "metrics/particle/class_4/efficiency": 0.8516649603843689, "metrics/particle/class_4/pid_accuracy": 0.20616114139556885, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.45824846625328064, "metrics/particle/count/bias_mean": -0.2751634120941162, "metrics/particle/count/mae": 3.879085063934326, "metrics/particle/count/prediction_mean": 70.29019927978516, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.068669892847538, "metrics/particle/event/energy_response_mean": 1.0154707431793213, "metrics/particle/event/met_abs_error": 7.935426712036133, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06148412451148033, "metrics/particle/event/scalar_pt_response_mean": 1.0294595956802368, "metrics/particle/event/vector_pt_closure": 0.06552549451589584, "metrics/particle/matched/delta_eta_abs_mean": 0.02009357139468193, "metrics/particle/matched/delta_phi_abs_mean": 0.013747360557317734, "metrics/particle/matched/delta_r_mean": 0.02691064402461052, "metrics/particle/matched/energy_relative_abs_mean": 0.09648103266954422, "metrics/particle/matched/pid_accuracy": 0.9539657235145569, "metrics/particle/matched/pt_relative_abs_mean": 0.09180683642625809, "metrics/particle/matching/duplicate_fraction": 0.044874656945466995, "metrics/particle/matching/efficiency": 0.7806696891784668, "metrics/particle/matching/f1": 0.7821947336196899, "metrics/particle/matching/purity": 0.7837257385253906, "metrics/particle/matching/target_coverage_dr0p05": 0.6525818705558777, "metrics/particle/matching/target_coverage_dr0p10": 0.819987952709198, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6787847876548767, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8322048783302307, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02827807329595089, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09210013598203659, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08973709493875504, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013302593491971493, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14165085554122925, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0059024132788181305, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4580288827419281, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.039816249161958694, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8524940013885498, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7624.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3679368495941162, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3974967002868652, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5118921995162964, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3795150518417358, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15759429335594177, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.272029876708984, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020591232925653458, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.998812198638916, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02798103541135788, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.805135726928711, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9810.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7520962953567505, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.3035247325897217, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.733826637268066, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.7289910316467285, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.311787486076355, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.04551474750041962, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.244722366333008, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.358586072921753, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.20698356628418, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03239181265234947, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09936974197626114, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10300836712121964, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.022105440497398376, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15585476160049438, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027696793898940086, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1531895399093628, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19278062880039215, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.955881357192993, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1372.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.938202977180481, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.189166307449341, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 79238.8984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 79238.734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31158560514450073, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3984625.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08975879848003387, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.214672803878784, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4510483145713806, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.69903564453125, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2529.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4990905821323395, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.6905945539474487, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.049668312072754, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.405404806137085, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06465538591146469, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.16101999580860138, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.860775470733643, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.8310753107070923, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.822349548339844, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02585073560476303, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09960655868053436, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0936950296163559, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01968276873230934, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14813801646232605, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007380463066510856, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14533580839633942, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.04583941772580147, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5748809576034546, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18969.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2006138414144516, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9314729571342468, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7532826662063599, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6264530420303345, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11835426092147827, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.58570671081543, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00552141759544611, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3602639436721802, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03901701420545578, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.811278223991394, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22458.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8296811580657959, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.552152156829834, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.963671684265137, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.887983322143555, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.2630383372306824, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3837395906448364, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.426751613616943, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8878309726715088, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.882235527038574, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02683340758085251, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09754849970340729, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09307292103767395, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018062222748994827, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1467905044555664, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0034686215221881866, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.28003084659576416, "metrics/particle/momentum/all/accepted/pz_bias": 0.032490324229002, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3883501291275024, "metrics/particle/momentum/all/accepted/samples": 27965.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.30139273405075073, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1542623043060303, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5759.89111328125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5759.759765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14346064627170563, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1074214.625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01589217409491539, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7477386593818665, "metrics/particle/momentum/all/assigned/pz_bias": 0.000288443174213171, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4204039573669434, "metrics/particle/momentum/all/assigned/samples": 34797.0, "metrics/particle/momentum/all/efficiency": 0.7822596430778503, "metrics/particle/momentum/all/event/energy_bias": 6.165082931518555, "metrics/particle/momentum/all/event/p3_error_mean": 20.29082679748535, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.659646987915039, "metrics/particle/momentum/all/event/px_bias": 0.5101704001426697, "metrics/particle/momentum/all/event/py_bias": 0.17720487713813782, "metrics/particle/momentum/all/event/pz_abs_mean": 14.760992050170898, "metrics/particle/momentum/all/event/pz_bias": 0.41534167528152466, "metrics/particle/momentum/all/event/pz_residual_rms": 22.76505470275879, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013173114508390427, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05789406970143318, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0504220686852932, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020103350281715393, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10051283240318298, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016688061878085136, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17425848543643951, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.024851148948073387, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.031962513923645, "metrics/particle/momentum/class_1/accepted/samples": 15580.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12031742930412292, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.40466928482055664, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5154544115066528, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.45428138971328735, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06896551698446274, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.13489818572998, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007876772433519363, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4239797294139862, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0045738741755485535, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.530557870864868, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9088257551193237, "metrics/particle/momentum/class_1/event/energy_bias": 9.199775695800781, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.01536750793457, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.350334167480469, "metrics/particle/momentum/class_1/event/px_bias": -0.5511870980262756, "metrics/particle/momentum/class_1/event/py_bias": 0.4956107437610626, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.812946319580078, "metrics/particle/momentum/class_1/event/pz_bias": 0.6429008841514587, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.936725616455078, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04369809478521347, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2091689258813858, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20952902734279633, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.046258166432380676, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24935561418533325, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04674610495567322, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.1070079803466797, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.20368750393390656, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.94084358215332, "metrics/particle/momentum/class_2/accepted/samples": 1091.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0369831323623657, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8658550381660461, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 78613.953125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 78613.46875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4032953977584839, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3968962.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08748529106378555, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.468438148498535, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3346825838088989, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.76475715637207, "metrics/particle/momentum/class_2/assigned/samples": 2549.0, "metrics/particle/momentum/class_2/efficiency": 0.37750864028930664, "metrics/particle/momentum/class_2/event/energy_bias": -6.835565567016602, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.150894165039062, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.684741020202637, "metrics/particle/momentum/class_2/event/px_bias": 0.36908864974975586, "metrics/particle/momentum/class_2/event/py_bias": -0.33953267335891724, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.819835662841797, "metrics/particle/momentum/class_2/event/pz_bias": -2.196042776107788, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.04344367980957, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0452810600399971, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14376400411128998, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14290128648281097, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0206486564129591, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18416984379291534, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014779234770685434, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23657599091529846, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.06243974342942238, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.887031614780426, "metrics/particle/momentum/class_3/accepted/samples": 10826.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3914557695388794, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0892109870910645, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0907435417175293, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9399595260620117, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1872042715549469, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.231945991516113, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012960296124219894, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6565619111061096, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0472719632089138, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.106959581375122, "metrics/particle/momentum/class_3/assigned/samples": 14583.0, "metrics/particle/momentum/class_3/efficiency": 0.712752640247345, "metrics/particle/momentum/class_3/event/energy_bias": 6.2706403732299805, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.550220489501953, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.677100658416748, "metrics/particle/momentum/class_3/event/px_bias": 0.677437424659729, "metrics/particle/momentum/class_3/event/py_bias": -0.15664276480674744, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.953214645385742, "metrics/particle/momentum/class_3/event/pz_bias": 1.7431617975234985, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.627815246582031, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01761968806385994, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11614222079515457, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11727368086576462, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0494127981364727, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17276684939861298, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013245033100247383, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7040967345237732, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1359603852033615, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6404061317443848, "metrics/particle/momentum/class_4/accepted/samples": 302.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.18848927319049835, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5361580848693848, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.744834303855896, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6642873287200928, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12747874855995178, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.826650619506836, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014164306223392487, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7764240503311157, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2085115760564804, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7680845260620117, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8531073331832886, "metrics/particle/momentum/class_4/event/energy_bias": -2.1080307960510254, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8505144119262695, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.029142379760742, "metrics/particle/momentum/class_4/event/px_bias": 0.04103950411081314, "metrics/particle/momentum/class_4/event/py_bias": 0.1546446979045868, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6183595657348633, "metrics/particle/momentum/class_4/event/pz_bias": 0.25442931056022644, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.470467567443848, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011750376783311367, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03787063807249069, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.037025682628154755, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02350912056863308, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08212769031524658, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2624189555644989, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.16024954617023468, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5707204341888428, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04066421464085579, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07483269274234772, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.07343776524066925, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0263533778488636, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.040462426841259, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.240997776389122, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.36246630549430847, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1258932650089264, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0258392095565796, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.3617382049560547, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.317706823348999, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9778413772583008, "metrics/particle/momentum/class_5/event/px_bias": -0.026208102703094482, "metrics/particle/momentum/class_5/event/py_bias": 0.023124855011701584, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7307206392288208, "metrics/particle/momentum/class_5/event/pz_bias": -0.029107525944709778, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.610875606536865, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.1910557746887207, "step_valid_time": 2052.806178331375, "step_total_time": 2053.997234582901, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0118318359088099, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12264705097690454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3032, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2853, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2832, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9363308171972432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9409630606860159, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9386412238855075, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05903693931398417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9294387922546767, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9340369393139841, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9317321927948676, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9363308171972432, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011698687816678, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13378236090724938, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2287, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2102, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9293239683933274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9256668124180149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9274917853231106, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07433318758198514, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.922739244951712, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9191080017490162, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9209200438116101, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9293239683933274, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0170797790390367, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13558591433842937, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1184, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1174, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8983308042488619, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9038167938931297, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9010654490106544, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09618320610687023, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8907435508345979, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8961832061068702, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.893455098934551, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8983308042488619} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_11000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..898566746973b4138eaae754c581181c742901df --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5210306934444866, "train": {"Classification_binary": 3.2871649265289307, "Classification": 0.022453011944890022, "Regression_pt": 0.17108677327632904, "Regression_eta": 0.014279663562774658, "Regression_sin_phi": 0.00785268284380436, "Regression_cos_phi": 0.0069757625460624695, "Regression_energy": 0.18174973130226135, "Total": 9.985005378723145, "Total_unweighted": 3.6915625520050526}, "train_group": {}, "valid": {"Classification_binary": 3.295820871988932, "Classification": 0.023242421448230743, "Regression_pt": 0.1808981498082479, "Regression_eta": 0.014055419713258743, "Regression_sin_phi": 0.008044116199016571, "Regression_cos_phi": 0.008630605414509773, "Regression_energy": 0.18659051259358725, "Total": 10.30838712056478, "Total_unweighted": 3.717282097165783, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.023320574313402176, "diagnostic/loss/cld_pf/Classification_binary": 3.282092571258545, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0081985117867589, "diagnostic/loss/cld_pf/Regression_energy": 0.18513166904449463, "diagnostic/loss/cld_pf/Regression_eta": 0.013030564412474632, "diagnostic/loss/cld_pf/Regression_pt": 0.18073880672454834, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007398645393550396, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09533864259719849, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2279428392648697, "diagnostic/regression/cld_pf/energy_residual_mean": 0.0682193711400032, "diagnostic/regression/cld_pf/energy_residual_rms": 0.48305195569992065, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.06161163002252579, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.22418028116226196, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08877876400947571, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4799838662147522, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9136800765991211, "metrics/particle/class_1/pid_accuracy": 0.9817510843276978, "metrics/particle/class_2/efficiency": 0.36732369661331177, "metrics/particle/class_2/pid_accuracy": 0.6401985287666321, "metrics/particle/class_3/efficiency": 0.7387921810150146, "metrics/particle/class_3/pid_accuracy": 0.9781664609909058, "metrics/particle/class_4/efficiency": 0.8668012022972107, "metrics/particle/class_4/pid_accuracy": 0.20256111025810242, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.4725050926208496, "metrics/particle/count/bias_mean": 0.20130719244480133, "metrics/particle/count/mae": 3.8379085063934326, "metrics/particle/count/prediction_mean": 70.76667022705078, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06711292266845703, "metrics/particle/event/energy_response_mean": 1.0019617080688477, "metrics/particle/event/met_abs_error": 7.613249778747559, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05816961079835892, "metrics/particle/event/scalar_pt_response_mean": 1.0138795375823975, "metrics/particle/event/vector_pt_closure": 0.06342046707868576, "metrics/particle/matched/delta_eta_abs_mean": 0.015557907521724701, "metrics/particle/matched/delta_phi_abs_mean": 0.01367918774485588, "metrics/particle/matched/delta_r_mean": 0.0231045912951231, "metrics/particle/matched/energy_relative_abs_mean": 0.0973086878657341, "metrics/particle/matched/pid_accuracy": 0.9568164348602295, "metrics/particle/matched/pt_relative_abs_mean": 0.09141600877046585, "metrics/particle/matching/duplicate_fraction": 0.0428638719022274, "metrics/particle/matching/efficiency": 0.79531329870224, "metrics/particle/matching/f1": 0.7941805124282837, "metrics/particle/matching/purity": 0.7930508852005005, "metrics/particle/matching/target_coverage_dr0p05": 0.743565022945404, "metrics/particle/matching/target_coverage_dr0p10": 0.8361876606941223, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.682267427444458, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8385124802589417, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025381948798894882, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09584606438875198, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09343703836202621, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.027866115793585777, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14835330843925476, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005751533899456263, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4510958194732666, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.014245427213609219, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7887216806411743, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7824.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3416767716407776, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4417729377746582, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3756415843963623, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.259164571762085, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15541362762451172, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.13595199584961, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018653690814971924, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9803240299224854, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04461759701371193, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6009013652801514, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9864.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7718259692192078, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.0471630096435547, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.043054580688477, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.332660675048828, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.0701361894607544, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3336489200592041, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.649971008300781, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.6487266421318054, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.499265670776367, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.033431943506002426, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10824363678693771, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10805771499872208, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03864040598273277, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16422836482524872, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0614752769470215, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.058029353618621826, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2302675247192383, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1422.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8809860348701477, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6475287675857544, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10560.9287109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10560.7685546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31928181648254395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 534370.3125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09250585734844208, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2470593452453613, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.21456614136695862, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.486990928649902, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2562.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5172790288925171, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.37570884823799133, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.684449195861816, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5867997407913208, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09846904873847961, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06099588796496391, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.481173515319824, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.1622202396392822, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.954971313476562, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02124248445034027, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09864544123411179, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09142276644706726, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012599327601492405, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14671720564365387, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006228589336387813, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13486944139003754, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.029010232537984848, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5779101252555847, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19266.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18960191309452057, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8185353875160217, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6386173367500305, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5048948526382446, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11274618655443192, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.658836364746094, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004712577443569899, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3459628224372864, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.052004165947437286, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7987642288208008, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22493.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8426715731620789, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0854841470718384, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.648133277893066, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.916112899780273, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.3977639377117157, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.0630738735198975, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.629961967468262, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.8189467191696167, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.014503479003906, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022986331954598427, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0983559638261795, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0928051546216011, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018087463453412056, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.148087739944458, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032617845572531223, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2678586542606354, "metrics/particle/momentum/all/accepted/pz_bias": 0.020617622882127762, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2743914127349854, "metrics/particle/momentum/all/accepted/samples": 28512.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.28328707814216614, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0554120540618896, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 775.6531982421875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 775.5224609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13995246589183807, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 144744.140625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01509207021445036, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7380114793777466, "metrics/particle/momentum/all/assigned/pz_bias": 0.030359381809830666, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3214051723480225, "metrics/particle/momentum/all/assigned/samples": 34919.0, "metrics/particle/momentum/all/efficiency": 0.7975607514381409, "metrics/particle/momentum/all/event/energy_bias": 2.7569382190704346, "metrics/particle/momentum/all/event/p3_error_mean": 19.883581161499023, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.393789291381836, "metrics/particle/momentum/all/event/px_bias": -0.5663691759109497, "metrics/particle/momentum/all/event/py_bias": -1.4577187299728394, "metrics/particle/momentum/all/event/pz_abs_mean": 14.59748649597168, "metrics/particle/momentum/all/event/pz_bias": 3.3324403762817383, "metrics/particle/momentum/all/event/pz_residual_rms": 23.008625030517578, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014189901761710644, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05645342543721199, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04650348424911499, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0125347301363945, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09793928265571594, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002111324341967702, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1757863163948059, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.017820952460169792, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9353839755058289, "metrics/particle/momentum/class_1/accepted/samples": 15630.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11240319907665253, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33561864495277405, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.41173022985458374, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.34714096784591675, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06608675420284271, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.544618606567383, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007822990417480469, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4153435528278351, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.013845322653651237, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.401458501815796, "metrics/particle/momentum/class_1/assigned/samples": 17129.0, "metrics/particle/momentum/class_1/efficiency": 0.9117423892021179, "metrics/particle/momentum/class_1/event/energy_bias": 5.785514831542969, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.593761444091797, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.618338584899902, "metrics/particle/momentum/class_1/event/px_bias": 0.2588121294975281, "metrics/particle/momentum/class_1/event/py_bias": -0.3636903166770935, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.864043235778809, "metrics/particle/momentum/class_1/event/pz_bias": 0.09286156296730042, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.84610939025879, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04218191280961037, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20909495651721954, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20967397093772888, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0608520433306694, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24848122894763947, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04021937772631645, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.033836841583252, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2003057599067688, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.765562057495117, "metrics/particle/momentum/class_2/accepted/samples": 1094.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0342223644256592, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8753721117973328, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10420.1435546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10419.611328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4318181872367859, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 530859.4375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08436055481433868, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4660704135894775, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.21781636774539948, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.503763675689697, "metrics/particle/momentum/class_2/assigned/samples": 2596.0, "metrics/particle/momentum/class_2/efficiency": 0.37854671478271484, "metrics/particle/momentum/class_2/event/energy_bias": -8.973786354064941, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.709747314453125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.225040435791016, "metrics/particle/momentum/class_2/event/px_bias": -0.1785060465335846, "metrics/particle/momentum/class_2/event/py_bias": -0.449581503868103, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.791889190673828, "metrics/particle/momentum/class_2/event/pz_bias": 0.376514196395874, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.123817443847656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03358650952577591, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14625416696071625, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14592397212982178, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03288603946566582, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1877278983592987, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008833922329358757, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2117392122745514, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.044975049793720245, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6894513368606567, "metrics/particle/momentum/class_3/accepted/samples": 11320.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.35476911067962646, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.932357668876648, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8239479064941406, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6843756437301636, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17638234794139862, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.117372512817383, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011317924596369267, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6350961327552795, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.09288232773542404, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0588266849517822, "metrics/particle/momentum/class_3/assigned/samples": 14667.0, "metrics/particle/momentum/class_3/efficiency": 0.7452762126922607, "metrics/particle/momentum/class_3/event/energy_bias": 8.251445770263672, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.766295433044434, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.578676223754883, "metrics/particle/momentum/class_3/event/px_bias": -0.6524066925048828, "metrics/particle/momentum/class_3/event/py_bias": -0.8149679899215698, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.35598373413086, "metrics/particle/momentum/class_3/event/pz_bias": 2.598483085632324, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.362932205200195, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01711784116923809, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11012405157089233, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11127473413944244, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03930054232478142, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16385418176651, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.019801979884505272, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7728047966957092, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.008465191349387169, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.7939751148223877, "metrics/particle/momentum/class_4/accepted/samples": 303.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2104691118001938, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3568015098571777, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4831715822219849, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4151743650436401, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11016949266195297, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.757840156555176, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01977401040494442, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8235421180725098, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.04184289649128914, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.8358640670776367, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8559321761131287, "metrics/particle/momentum/class_4/event/energy_bias": -1.8609708547592163, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.018217086791992, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0983548164367676, "metrics/particle/momentum/class_4/event/px_bias": -0.01731395721435547, "metrics/particle/momentum/class_4/event/py_bias": 0.2356421947479248, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7762051820755005, "metrics/particle/momentum/class_4/event/pz_bias": 0.14191776514053345, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.139185428619385, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012515891343355179, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02571723237633705, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025770146399736404, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013248978182673454, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06625951826572418, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2035161405801773, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1326671540737152, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.48403748869895935, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.023133553564548492, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06043446436524391, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05998734384775162, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.023917196318507195, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.18047299981117249, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2993960380554199, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.0652984082698822, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.033927083015442, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": -0.4452648460865021, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9874396920204163, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7845456600189209, "metrics/particle/momentum/class_5/event/px_bias": 0.023045416921377182, "metrics/particle/momentum/class_5/event/py_bias": -0.06512115895748138, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4805881679058075, "metrics/particle/momentum/class_5/event/pz_bias": 0.12266357243061066, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.8198206424713135, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.250457525253296, "step_valid_time": 2237.722925424576, "step_total_time": 2238.9733831882477, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004906542047428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.108896677239668, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2859, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9455201837873318, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9476973684210527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466075242319698, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05230263157894737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9382999671808336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9404605263157895, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9393790044356826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9455201837873318, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040843376355189, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12003667799778706, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2108, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2095, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9253731343283582, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9274087109546855, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9263898044385849, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07259128904531456, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9196663740122915, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9216893972723273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9206767743353109, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9253731343283582, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071446060451525, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13132950350088823, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1193, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9112291350531108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9133079847908745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9122673756171668, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08669201520912548, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9051593323216995, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9072243346007605, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9061906570451956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9112291350531108} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_12000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..e20115e694e8c8da5e4c29619031d3fcdc64259e --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5683971201212581, "train": {"Classification_binary": 3.3008885383605957, "Classification": 0.02283535711467266, "Regression_pt": 0.1824522167444229, "Regression_eta": 0.01634019799530506, "Regression_sin_phi": 0.0097350450232625, "Regression_cos_phi": 0.008370253257453442, "Regression_energy": 0.19138821959495544, "Total": 10.614089012145996, "Total_unweighted": 3.7320098280906677}, "train_group": {}, "valid": {"Classification_binary": 3.286155382792155, "Classification": 0.022983849048614502, "Regression_pt": 0.17988693714141846, "Regression_eta": 0.013550717383623123, "Regression_sin_phi": 0.00798320397734642, "Regression_cos_phi": 0.008664869392911593, "Regression_energy": 0.18620808919270834, "Total": 10.246740341186523, "Total_unweighted": 3.7054330489287772, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.02308376133441925, "diagnostic/loss/cld_pf/Classification_binary": 3.2672643661499023, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.008241323754191399, "diagnostic/loss/cld_pf/Regression_energy": 0.18679411709308624, "diagnostic/loss/cld_pf/Regression_eta": 0.01255412120372057, "diagnostic/loss/cld_pf/Regression_pt": 0.18202124536037445, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007357608526945114, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08364646881818771, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.22155839204788208, "diagnostic/regression/cld_pf/energy_residual_mean": 0.056527186185121536, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4728335440158844, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.050860803574323654, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.21749991178512573, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07802794128656387, "diagnostic/regression/cld_pf/pt_residual_rms": 0.46894437074661255, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.91977459192276, "metrics/particle/class_1/pid_accuracy": 0.9806634187698364, "metrics/particle/class_2/efficiency": 0.36823517084121704, "metrics/particle/class_2/pid_accuracy": 0.6503713130950928, "metrics/particle/class_3/efficiency": 0.7244637608528137, "metrics/particle/class_3/pid_accuracy": 0.9784626364707947, "metrics/particle/class_4/efficiency": 0.871846616268158, "metrics/particle/class_4/pid_accuracy": 0.21064814925193787, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.5476673245429993, "metrics/particle/count/bias_mean": -0.9686274528503418, "metrics/particle/count/mae": 3.9947712421417236, "metrics/particle/count/prediction_mean": 69.59673309326172, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06841795891523361, "metrics/particle/event/energy_response_mean": 1.0006517171859741, "metrics/particle/event/met_abs_error": 7.716114044189453, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05754033848643303, "metrics/particle/event/scalar_pt_response_mean": 1.0108033418655396, "metrics/particle/event/vector_pt_closure": 0.06549342721700668, "metrics/particle/matched/delta_eta_abs_mean": 0.01229909062385559, "metrics/particle/matched/delta_phi_abs_mean": 0.014887717552483082, "metrics/particle/matched/delta_r_mean": 0.021525561809539795, "metrics/particle/matched/energy_relative_abs_mean": 0.09474039077758789, "metrics/particle/matched/pid_accuracy": 0.9570645093917847, "metrics/particle/matched/pt_relative_abs_mean": 0.08923771232366562, "metrics/particle/matching/duplicate_fraction": 0.04176253452897072, "metrics/particle/matching/efficiency": 0.7923586368560791, "metrics/particle/matching/f1": 0.7978344559669495, "metrics/particle/matching/purity": 0.8033864498138428, "metrics/particle/matching/target_coverage_dr0p05": 0.731709361076355, "metrics/particle/matching/target_coverage_dr0p10": 0.8298615217208862, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6846848726272583, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8339647054672241, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02387678436934948, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0898023471236229, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08793617784976959, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02194696106016636, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14103670418262482, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005157962441444397, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4066445827484131, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01196274347603321, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.66584312915802, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7755.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.34717297554016113, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.333561897277832, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.447379469871521, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3285291194915771, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14802631735801697, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.139562606811523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02097039483487606, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9633017182350159, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.016601279377937317, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5289838314056396, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9728.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7650192379951477, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.892825722694397, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.407245635986328, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.630898952484131, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6299865245819092, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.8157529830932617, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.970959663391113, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.9906388521194458, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.890840530395508, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03142600506544113, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09933070093393326, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10193674266338348, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.029018938541412354, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15496399998664856, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023874487727880478, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0142812728881836, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20228354632854462, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3537404537200928, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1466.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8710318803787231, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.0707013607025146, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 37443.59375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 37443.43359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.29651859402656555, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1871604.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08843537420034409, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2000539302825928, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.437076598405838, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.84708309173584, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2499.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5332848429679871, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.1866873502731323, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.049596786499023, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3878436088562012, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.06456497311592102, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07297994941473007, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.864555358886719, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5552865862846375, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.5894832611084, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019440021365880966, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09792222827672958, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09105418622493744, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017375845462083817, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14688454568386078, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008873577462509274, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13070887327194214, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010063160210847855, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6096160411834717, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19158.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.19144588708877563, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8554266095161438, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6241819858551025, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4950692653656006, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11276176571846008, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.049161911010742, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005324861500412226, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3484687805175781, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.032272323966026306, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7676966190338135, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22348.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8379477858543396, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7634053826332092, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.899821281433105, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.280016899108887, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.15760110318660736, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.1408820152282715, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.524127006530762, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6601228713989258, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.442876815795898, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021271605044603348, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09577609598636627, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09076431393623352, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019226431846618652, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1457427591085434, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032418337650597095, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25175607204437256, "metrics/particle/momentum/all/accepted/pz_bias": -0.006925146095454693, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2610418796539307, "metrics/particle/momentum/all/accepted/samples": 28379.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.28438007831573486, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.077791690826416, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2707.14599609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2707.01708984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1359652876853943, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 503171.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015733912587165833, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.727563738822937, "metrics/particle/momentum/all/assigned/pz_bias": -0.006060308311134577, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.34378981590271, "metrics/particle/momentum/all/assigned/samples": 34575.0, "metrics/particle/momentum/all/efficiency": 0.7938404083251953, "metrics/particle/momentum/all/event/energy_bias": 2.8429183959960938, "metrics/particle/momentum/all/event/p3_error_mean": 20.134986877441406, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.779812812805176, "metrics/particle/momentum/all/event/px_bias": 0.8521525859832764, "metrics/particle/momentum/all/event/py_bias": -4.02961540222168, "metrics/particle/momentum/all/event/pz_abs_mean": 14.506239891052246, "metrics/particle/momentum/all/event/pz_bias": 0.22477057576179504, "metrics/particle/momentum/all/event/pz_residual_rms": 23.070255279541016, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012309958226978779, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05594033747911453, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04628019034862518, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011382059194147587, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09788955003023148, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019036740995943546, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15453772246837616, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0036574439145624638, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8835944533348083, "metrics/particle/momentum/class_1/accepted/samples": 15759.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10514822602272034, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3129585385322571, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.39555680751800537, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.33048707246780396, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05836689472198486, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.631063461303711, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0075293295085430145, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.37886595726013184, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0008731976267881691, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.470810651779175, "metrics/particle/momentum/class_1/assigned/samples": 17133.0, "metrics/particle/momentum/class_1/efficiency": 0.9192673563957214, "metrics/particle/momentum/class_1/event/energy_bias": 1.8343713283538818, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.315642356872559, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.48318862915039, "metrics/particle/momentum/class_1/event/px_bias": 0.6661306619644165, "metrics/particle/momentum/class_1/event/py_bias": -0.803338885307312, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.626326560974121, "metrics/particle/momentum/class_1/event/pz_bias": -0.1539963334798813, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.747426986694336, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03909264877438545, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20701707899570465, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21047955751419067, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.051185425370931625, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2477513998746872, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.041208792477846146, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.013606071472168, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1745612472295761, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.477285385131836, "metrics/particle/momentum/class_2/accepted/samples": 1092.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0525258779525757, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8902899026870728, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 37172.953125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 37172.44921875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4100119173526764, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1864899.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08939213305711746, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.559058904647827, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09045030921697617, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.794919967651367, "metrics/particle/momentum/class_2/assigned/samples": 2517.0, "metrics/particle/momentum/class_2/efficiency": 0.37785467505455017, "metrics/particle/momentum/class_2/event/energy_bias": -4.329154968261719, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.951980590820312, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.78320598602295, "metrics/particle/momentum/class_2/event/px_bias": 0.21476665139198303, "metrics/particle/momentum/class_2/event/py_bias": -2.0472114086151123, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.599382400512695, "metrics/particle/momentum/class_2/event/pz_bias": -0.1529020369052887, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.751474380493164, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03256560489535332, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14223411679267883, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14274413883686066, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03672447055578232, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18448367714881897, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010850890539586544, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20661966502666473, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006852223537862301, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9261214137077332, "metrics/particle/momentum/class_3/accepted/samples": 11059.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36899083852767944, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0250818729400635, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.938729166984558, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7972112894058228, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1819002628326416, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.6911039352417, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012571190483868122, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6502068042755127, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0012329751625657082, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.9592487812042236, "metrics/particle/momentum/class_3/assigned/samples": 14398.0, "metrics/particle/momentum/class_3/efficiency": 0.7280926704406738, "metrics/particle/momentum/class_3/event/energy_bias": 6.873990058898926, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.63127326965332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.648911952972412, "metrics/particle/momentum/class_3/event/px_bias": -0.14394918084144592, "metrics/particle/momentum/class_3/event/py_bias": -1.506561279296875, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.065291404724121, "metrics/particle/momentum/class_3/event/pz_bias": 0.03722301870584488, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.891427993774414, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0170180331915617, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10857421159744263, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11125205457210541, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.047163140028715134, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16092291474342346, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016393441706895828, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6390208601951599, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.02312876656651497, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.141711711883545, "metrics/particle/momentum/class_4/accepted/samples": 305.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1723056584596634, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3972562551498413, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5684056282043457, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4953502416610718, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11864406615495682, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.929579734802246, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.025423727929592133, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8385223746299744, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.17120864987373352, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.815725803375244, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8615819215774536, "metrics/particle/momentum/class_4/event/energy_bias": -1.603187084197998, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0229179859161377, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.121581554412842, "metrics/particle/momentum/class_4/event/px_bias": -0.01708114892244339, "metrics/particle/momentum/class_4/event/py_bias": 0.2469540685415268, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.775571584701538, "metrics/particle/momentum/class_4/event/pz_bias": 0.1832665503025055, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.754128932952881, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010070468299090862, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026350373402237892, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024927491322159767, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009944044053554535, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05398082360625267, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18573297560214996, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0315791554749012, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5561409592628479, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04625360295176506, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05844249203801155, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05669549107551575, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008364234119653702, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1573115885257721, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.27594879269599915, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.056454967707395554, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9567701816558838, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": 0.06689884513616562, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.171499252319336, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9032473564147949, "metrics/particle/momentum/class_5/event/px_bias": 0.13228558003902435, "metrics/particle/momentum/class_5/event/py_bias": 0.0805426687002182, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6132887601852417, "metrics/particle/momentum/class_5/event/pz_bias": 0.31117939949035645, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.4874587059020996, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.2781262397766113, "step_valid_time": 2418.476496219635, "step_total_time": 2419.75462269783, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002513477806707, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1132664192767362, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9445356087955366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9498349834983498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9471778838242554, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050165016501650166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9392845421726288, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9445544554455445, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9419121276945861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9445356087955366, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0075855407326282, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13175963850407102, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2263, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9411764705882353, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9474149359257623, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9442853996916979, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052585064074237735, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.932396839332748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9385771100309324, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354767672318872, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9411764705882353, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0104649131314865, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1307829423065051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1307, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9142640364188164, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9219586840091814, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9180952380952381, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07804131599081866, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9081942336874052, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9158377964804897, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.912, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9142640364188164} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_13000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..6af5ea5228543241df862b840b1e8760031de3fb --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6157635467980296, "train": {"Classification_binary": 3.193202495574951, "Classification": 0.0222002025693655, "Regression_pt": 0.1661701798439026, "Regression_eta": 0.012930752709507942, "Regression_sin_phi": 0.010616745799779892, "Regression_cos_phi": 0.006709885783493519, "Regression_energy": 0.17299793660640717, "Total": 9.911617279052734, "Total_unweighted": 3.584828198887408}, "train_group": {}, "valid": {"Classification_binary": 3.214768727620443, "Classification": 0.022853342195351917, "Regression_pt": 0.1761219302813212, "Regression_eta": 0.013347250719865164, "Regression_sin_phi": 0.007804582516352336, "Regression_cos_phi": 0.008389120921492577, "Regression_energy": 0.18239239851633707, "Total": 10.04011599222819, "Total_unweighted": 3.625677352771163, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.023112639784812927, "diagnostic/loss/cld_pf/Classification_binary": 3.208113193511963, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007988077588379383, "diagnostic/loss/cld_pf/Regression_energy": 0.18018114566802979, "diagnostic/loss/cld_pf/Regression_eta": 0.012293371371924877, "diagnostic/loss/cld_pf/Regression_pt": 0.17533816397190094, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007227267138659954, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08884397149085999, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2238559126853943, "diagnostic/regression/cld_pf/energy_residual_mean": 0.061724696308374405, "diagnostic/regression/cld_pf/energy_residual_rms": 0.47233882546424866, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.052335627377033234, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2182806432247162, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07950276136398315, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4675833582878113, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9204262495040894, "metrics/particle/class_1/pid_accuracy": 0.9830092191696167, "metrics/particle/class_2/efficiency": 0.37917283177375793, "metrics/particle/class_2/pid_accuracy": 0.6640625, "metrics/particle/class_3/efficiency": 0.737429678440094, "metrics/particle/class_3/pid_accuracy": 0.9778280854225159, "metrics/particle/class_4/efficiency": 0.849646806716919, "metrics/particle/class_4/pid_accuracy": 0.22090260684490204, "metrics/particle/class_5/efficiency": 0.9592233300209045, "metrics/particle/class_5/pid_accuracy": 0.5101214647293091, "metrics/particle/count/bias_mean": 0.2908496856689453, "metrics/particle/count/mae": 3.7862744331359863, "metrics/particle/count/prediction_mean": 70.85620880126953, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06554684787988663, "metrics/particle/event/energy_response_mean": 1.000673532485962, "metrics/particle/event/met_abs_error": 7.121297359466553, "metrics/particle/event/scalar_pt_relative_abs_error": 0.056526314467191696, "metrics/particle/event/scalar_pt_response_mean": 1.007076621055603, "metrics/particle/event/vector_pt_closure": 0.060518842190504074, "metrics/particle/matched/delta_eta_abs_mean": 0.01335995364934206, "metrics/particle/matched/delta_phi_abs_mean": 0.012268733233213425, "metrics/particle/matched/delta_r_mean": 0.020269958302378654, "metrics/particle/matched/energy_relative_abs_mean": 0.09847528487443924, "metrics/particle/matched/pid_accuracy": 0.958537220954895, "metrics/particle/matched/pt_relative_abs_mean": 0.09110035002231598, "metrics/particle/matching/duplicate_fraction": 0.04023614153265953, "metrics/particle/matching/efficiency": 0.7988329529762268, "metrics/particle/matching/f1": 0.7971900701522827, "metrics/particle/matching/purity": 0.7955539226531982, "metrics/particle/matching/target_coverage_dr0p05": 0.7544574737548828, "metrics/particle/matching/target_coverage_dr0p10": 0.8373639583587646, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6859630346298218, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8433473706245422, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02220379188656807, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09511657804250717, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09179173409938812, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018406080082058907, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1461428999900818, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005777442827820778, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4206992983818054, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0043435185216367245, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.724892020225525, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7962.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32170727849006653, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3691611289978027, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3456461429595947, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2185138463974, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14625781774520874, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.836482048034668, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020274359732866287, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9698509573936462, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.017161961644887924, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.8568105697631836, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9914.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7854394912719727, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.2775840759277344, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.211282730102539, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.05592679977417, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3349985182285309, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3361806571483612, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.911914825439453, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.6664010286331177, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.554851531982422, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03224031999707222, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10352741181850433, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10616528987884521, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.029625069350004196, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15990082919597626, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021476510912179947, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0179203748703003, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.043325167149305344, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.192319631576538, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1490.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8349688053131104, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6348201036453247, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 78748.2890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 78748.1171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2956218421459198, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4000531.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08020146936178207, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1220169067382812, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.10318595170974731, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.705081939697266, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2581.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5420152544975281, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.914972186088562, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.333839416503906, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.309594988822937, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.22406144440174103, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.046746719628572464, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.161711692810059, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.035266757011413574, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.434284210205078, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018912596628069878, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10162165015935898, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09251103550195694, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016105782240629196, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1497529000043869, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005207791109569371, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13620354235172272, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.011623984202742577, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5530785322189331, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19202.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.19223949313163757, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.175799012184143, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8806944489479065, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7488080263137817, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11251001060009003, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 44.23157501220703, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004671234171837568, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3434315025806427, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013406174257397652, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8598719835281372, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22478.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8398723006248474, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9504944086074829, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.951119422912598, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.436750888824463, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.003575839102268219, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.5910059809684753, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.2025957107543945, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.5025933980941772, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.944282531738281, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020520150661468506, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09991320967674255, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09302118420600891, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017447957769036293, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1493067890405655, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030711244326084852, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2611045241355896, "metrics/particle/momentum/all/accepted/pz_bias": 0.004329802468419075, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.249655842781067, "metrics/particle/momentum/all/accepted/samples": 28654.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2763737440109253, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2644883394241333, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5812.5546875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5812.421875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1355903148651123, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1086789.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014668458141386509, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7260653376579285, "metrics/particle/momentum/all/assigned/pz_bias": 0.02109657973051071, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4687745571136475, "metrics/particle/momentum/all/assigned/samples": 34973.0, "metrics/particle/momentum/all/efficiency": 0.8015329241752625, "metrics/particle/momentum/all/event/energy_bias": 2.2420616149902344, "metrics/particle/momentum/all/event/p3_error_mean": 18.372053146362305, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.703879356384277, "metrics/particle/momentum/all/event/px_bias": 0.5554841160774231, "metrics/particle/momentum/all/event/py_bias": 0.3015720546245575, "metrics/particle/momentum/all/event/pz_abs_mean": 13.34092903137207, "metrics/particle/momentum/all/event/pz_bias": 2.204261302947998, "metrics/particle/momentum/all/event/pz_residual_rms": 22.045801162719727, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01130545511841774, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05851591378450394, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0462813675403595, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00470257131382823, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09947666525840759, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017747353995218873, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17167294025421143, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.014964668080210686, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9363553524017334, "metrics/particle/momentum/class_1/accepted/samples": 15777.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10427871346473694, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.333048939704895, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.39899635314941406, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3301140069961548, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05992181599140167, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.811866760253906, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006534803658723831, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3822028338909149, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.004502890165895224, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3502554893493652, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9203173518180847, "metrics/particle/momentum/class_1/event/energy_bias": 5.076871395111084, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.494357109069824, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.580503463745117, "metrics/particle/momentum/class_1/event/px_bias": 0.7861791849136353, "metrics/particle/momentum/class_1/event/py_bias": -0.33225560188293457, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.737915992736816, "metrics/particle/momentum/class_1/event/pz_bias": 0.06558986008167267, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.885414123535156, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04232439771294594, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20404580235481262, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20677193999290466, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05713428184390068, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24464210867881775, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04035874456167221, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.933648705482483, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.39099231362342834, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.580511569976807, "metrics/particle/momentum/class_2/accepted/samples": 1115.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0445983409881592, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9517579674720764, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 77633.4765625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 77632.9453125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40909090638160706, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3972161.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08861726522445679, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.6042933464050293, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.061021529138088226, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.618934631347656, "metrics/particle/momentum/class_2/assigned/samples": 2618.0, "metrics/particle/momentum/class_2/efficiency": 0.3858131468296051, "metrics/particle/momentum/class_2/event/energy_bias": -8.338607788085938, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.840126037597656, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.899351119995117, "metrics/particle/momentum/class_2/event/px_bias": -0.12978993356227875, "metrics/particle/momentum/class_2/event/py_bias": 0.30761078000068665, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.152273178100586, "metrics/particle/momentum/class_2/event/pz_bias": 0.14366984367370605, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.423608779907227, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.031522080302238464, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14821624755859375, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14755666255950928, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0422675646841526, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18978141248226166, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010620409157127142, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21132513880729675, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.025412781164050102, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7109675407409668, "metrics/particle/momentum/class_3/accepted/samples": 11299.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.34511393308639526, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.418360710144043, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0906524658203125, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9512635469436646, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17645856738090515, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 55.868778228759766, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011028660461306572, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6180905699729919, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.05066491290926933, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.959763765335083, "metrics/particle/momentum/class_3/assigned/samples": 14689.0, "metrics/particle/momentum/class_3/efficiency": 0.7438936233520508, "metrics/particle/momentum/class_3/event/energy_bias": 7.773176670074463, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.612421989440918, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.226838111877441, "metrics/particle/momentum/class_3/event/px_bias": -0.01357896625995636, "metrics/particle/momentum/class_3/event/py_bias": 0.07459185272455215, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.3025007247924805, "metrics/particle/momentum/class_3/event/pz_bias": 1.4196202754974365, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.040202140808105, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014945248141884804, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10967619717121124, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11057480424642563, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03500690311193466, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1657911092042923, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010101010091602802, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6469918489456177, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.053644899278879166, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.4379069805145264, "metrics/particle/momentum/class_4/accepted/samples": 297.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.18826057016849518, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3832495212554932, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3599748611450195, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.278610110282898, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.13276836276054382, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.418139457702637, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7664535641670227, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.18943290412425995, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.8038077354431152, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8389830589294434, "metrics/particle/momentum/class_4/event/energy_bias": -1.8768641948699951, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.9160847663879395, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.038008213043213, "metrics/particle/momentum/class_4/event/px_bias": -0.14592045545578003, "metrics/particle/momentum/class_4/event/py_bias": 0.2569575905799866, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7056101560592651, "metrics/particle/momentum/class_4/event/pz_bias": 0.28132838010787964, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.486886978149414, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010962462984025478, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.029681634157896042, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027793461456894875, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008962437510490417, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06919953972101212, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2245054543018341, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1256231814622879, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5154200196266174, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04396119341254234, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.058652956038713455, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0558042973279953, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010227258317172527, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17573495209217072, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.3214469850063324, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.052681080996990204, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0642222166061401, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.39251410961151123, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1009448766708374, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8396478295326233, "metrics/particle/momentum/class_5/event/px_bias": 0.05859420821070671, "metrics/particle/momentum/class_5/event/py_bias": -0.005332573316991329, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5753476619720459, "metrics/particle/momentum/class_5/event/pz_bias": 0.29405301809310913, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.2797815799713135, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3289861679077148, "step_valid_time": 2597.0080666542053, "step_total_time": 2598.3370530605316, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007345207433231, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1104045207850407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2860, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9442074171316048, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9491916859122402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466929911154985, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050808314087759814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9386281588447654, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9435829759155394, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9410990457387298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9442074171316048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042732802271386, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12021458553402309, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9376646180860404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9393139841688655, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9384885764499121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06068601583113457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.932396839332748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9340369393139841, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9332161687170475, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9376646180860404, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0103248581832371, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12937698002015863, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1309, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9142640364188164, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.920550038197097, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9173962695089456, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07944996180290298, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9081942336874052, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9144385026737968, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9113056718690522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9142640364188164} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_14000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..70a5310be18334ed81a34e2d28df35ba2325d660 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6631299734748011, "train": {"Classification_binary": 3.0044710636138916, "Classification": 0.021966246888041496, "Regression_pt": 0.1806139349937439, "Regression_eta": 0.012761649675667286, "Regression_sin_phi": 0.009193578734993935, "Regression_cos_phi": 0.006945899222046137, "Regression_energy": 0.18924494087696075, "Total": 9.936402320861816, "Total_unweighted": 3.425197314005345}, "train_group": {}, "valid": {"Classification_binary": 3.1496191024780273, "Classification": 0.022425199548403423, "Regression_pt": 0.1757738987604777, "Regression_eta": 0.013233357419570288, "Regression_sin_phi": 0.00773331585029761, "Regression_cos_phi": 0.008287020027637482, "Regression_energy": 0.18221598863601685, "Total": 9.934335072835287, "Total_unweighted": 3.559287882720431, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.02242754027247429, "diagnostic/loss/cld_pf/Classification_binary": 3.1385958194732666, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007764047477394342, "diagnostic/loss/cld_pf/Regression_energy": 0.1826055645942688, "diagnostic/loss/cld_pf/Regression_eta": 0.012451200745999813, "diagnostic/loss/cld_pf/Regression_pt": 0.1773127019405365, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007102061063051224, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09772009402513504, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.22339153289794922, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07060081511735916, "diagnostic/regression/cld_pf/energy_residual_rms": 0.47424083948135376, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.06245528161525726, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2193945199251175, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08962241560220718, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4688274562358856, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9189121723175049, "metrics/particle/class_1/pid_accuracy": 0.9829603433609009, "metrics/particle/class_2/efficiency": 0.37883102893829346, "metrics/particle/class_2/pid_accuracy": 0.6673684120178223, "metrics/particle/class_3/efficiency": 0.7308588027954102, "metrics/particle/class_3/pid_accuracy": 0.9780797958374023, "metrics/particle/class_4/efficiency": 0.8526740670204163, "metrics/particle/class_4/pid_accuracy": 0.2153846174478531, "metrics/particle/class_5/efficiency": 0.9495145678520203, "metrics/particle/class_5/pid_accuracy": 0.4948875308036804, "metrics/particle/count/bias_mean": -0.5326797366142273, "metrics/particle/count/mae": 3.821568727493286, "metrics/particle/count/prediction_mean": 70.03267669677734, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06775035709142685, "metrics/particle/event/energy_response_mean": 0.9838504195213318, "metrics/particle/event/met_abs_error": 7.6869587898254395, "metrics/particle/event/scalar_pt_relative_abs_error": 0.054681193083524704, "metrics/particle/event/scalar_pt_response_mean": 1.0039186477661133, "metrics/particle/event/vector_pt_closure": 0.06107530742883682, "metrics/particle/matched/delta_eta_abs_mean": 0.014278207905590534, "metrics/particle/matched/delta_phi_abs_mean": 0.012105554342269897, "metrics/particle/matched/delta_r_mean": 0.020847806707024574, "metrics/particle/matched/energy_relative_abs_mean": 0.09668508917093277, "metrics/particle/matched/pid_accuracy": 0.9585152864456177, "metrics/particle/matched/pt_relative_abs_mean": 0.09208443760871887, "metrics/particle/matching/duplicate_fraction": 0.04058796167373657, "metrics/particle/matching/efficiency": 0.7952855229377747, "metrics/particle/matching/f1": 0.7982985973358154, "metrics/particle/matching/purity": 0.80133455991745, "metrics/particle/matching/target_coverage_dr0p05": 0.7499282360076904, "metrics/particle/matching/target_coverage_dr0p10": 0.839216411113739, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6836567521095276, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8364933133125305, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022861560806632042, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09255034476518631, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09052100777626038, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017686665058135986, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14342275261878967, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0059637101367115974, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.44146180152893066, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.019730115309357643, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7747560739517212, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7881.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3230552673339844, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.258226990699768, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.386311650276184, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2622807025909424, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14376084506511688, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.197088241577148, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0182634424418211, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9073124527931213, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04423488676548004, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2136948108673096, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9801.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7774489521980286, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.160754680633545, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.312968254089355, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.9388599395751953, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.03125663474202156, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.26306721568107605, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.162332534790039, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5033791661262512, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.019989013671875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03217846900224686, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10342386364936829, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10367383807897568, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.009516246616840363, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1572965383529663, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021066490560770035, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0299911499023438, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.02457989566028118, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6211118698120117, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1519.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7832384705543518, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3598719835281372, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 36534.2578125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 36534.078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2785009741783142, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1839310.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08047337085008621, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9808762073516846, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.02906491979956627, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.877728462219238, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2535.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5525645613670349, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.149809837341309, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.895400047302246, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2083998918533325, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05490870028734207, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.02511495351791382, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.748758316040039, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.8787501454353333, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.188493728637695, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019203737378120422, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09903839975595474, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09330596774816513, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.030495258048176765, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14879195392131805, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005753138102591038, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1340211182832718, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011317788623273373, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5902271270751953, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19120.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18451645970344543, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7924602031707764, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6289850473403931, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.513587474822998, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11850728839635849, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.596635341644287, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004458513576537371, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33591240644454956, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0062768543139100075, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6491718292236328, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22429.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8362857103347778, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.7291457653045654, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.28732681274414, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.857775688171387, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.7621757984161377, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.4181586503982544, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.137687683105469, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2674160301685333, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.114455223083496, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02090555988252163, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09747911989688873, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09308859705924988, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.025838466361165047, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1477976143360138, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003155680140480399, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.26669710874557495, "metrics/particle/momentum/all/accepted/pz_bias": -0.0034445980563759804, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.342501163482666, "metrics/particle/momentum/all/accepted/samples": 28520.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2672312259674072, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9651445150375366, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2664.8076171875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2664.685546875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1372932493686676, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 496675.5625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013893283903598785, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6898682713508606, "metrics/particle/momentum/all/assigned/pz_bias": 0.014400982297956944, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2269647121429443, "metrics/particle/momentum/all/assigned/samples": 34765.0, "metrics/particle/momentum/all/efficiency": 0.7977845668792725, "metrics/particle/momentum/all/event/energy_bias": -3.581418991088867, "metrics/particle/momentum/all/event/p3_error_mean": 19.10787582397461, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.097679138183594, "metrics/particle/momentum/all/event/px_bias": 1.8483411073684692, "metrics/particle/momentum/all/event/py_bias": 0.18020638823509216, "metrics/particle/momentum/all/event/pz_abs_mean": 13.816134452819824, "metrics/particle/momentum/all/event/pz_bias": 1.6495453119277954, "metrics/particle/momentum/all/event/pz_residual_rms": 21.863006591796875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010862699709832668, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.057115279138088226, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04955839365720749, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01843191497027874, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10100946575403214, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001460317405872047, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16920159757137299, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.019191481173038483, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9749172925949097, "metrics/particle/momentum/class_1/accepted/samples": 15750.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10166753828525543, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.34430137276649475, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.41010743379592896, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3501380980014801, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06260575354099274, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.006051063537598, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006884882226586342, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.39618486166000366, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0330248586833477, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5002970695495605, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9187423586845398, "metrics/particle/momentum/class_1/event/energy_bias": 6.959751129150391, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.739313125610352, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.69472599029541, "metrics/particle/momentum/class_1/event/px_bias": 1.0896824598312378, "metrics/particle/momentum/class_1/event/py_bias": -0.4593479037284851, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.944987297058105, "metrics/particle/momentum/class_1/event/pz_bias": -1.035614013671875, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.856943130493164, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.041440293192863464, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20782539248466492, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20725511014461517, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06479892134666443, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24695280194282532, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.047111112624406815, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.1002705097198486, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17455178499221802, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.092808246612549, "metrics/particle/momentum/class_2/accepted/samples": 1125.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.988081157207489, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8302880525588989, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 36232.3359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 36231.80859375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40140846371650696, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1831738.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08998434990644455, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4719080924987793, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.010998470708727837, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.598265171051025, "metrics/particle/momentum/class_2/assigned/samples": 2556.0, "metrics/particle/momentum/class_2/efficiency": 0.38927334547042847, "metrics/particle/momentum/class_2/event/energy_bias": -12.638896942138672, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.410337448120117, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.36575698852539, "metrics/particle/momentum/class_2/event/px_bias": 0.4248582720756531, "metrics/particle/momentum/class_2/event/py_bias": 0.4545894265174866, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.418261528015137, "metrics/particle/momentum/class_2/event/pz_bias": 0.3306122422218323, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.501426696777344, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03331544250249863, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14374180138111115, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14323976635932922, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04474394768476486, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18553388118743896, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008946944726631045, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20987167954444885, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03419482335448265, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7206753492355347, "metrics/particle/momentum/class_3/accepted/samples": 11177.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3398987054824829, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.712348461151123, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6904075145721436, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5633431673049927, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18022416532039642, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.534597396850586, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008939008228480816, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5516872406005859, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07559827715158463, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.694502115249634, "metrics/particle/momentum/class_3/assigned/samples": 14543.0, "metrics/particle/momentum/class_3/efficiency": 0.7358614802360535, "metrics/particle/momentum/class_3/event/energy_bias": 4.1944899559021, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.608896255493164, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.114241600036621, "metrics/particle/momentum/class_3/event/px_bias": 0.41717779636383057, "metrics/particle/momentum/class_3/event/py_bias": -0.15211085975170135, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.302439212799072, "metrics/particle/momentum/class_3/event/pz_bias": 2.150386333465576, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.403702735900879, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015090144239366055, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11806599050760269, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11902868002653122, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05757898464798927, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16770467162132263, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013201320543885231, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6666213274002075, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07511411607265472, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.157639265060425, "metrics/particle/momentum/class_4/accepted/samples": 303.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.20170965790748596, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7430689334869385, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6011569499969482, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5310022830963135, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12711864709854126, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.279796600341797, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6928450465202332, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.04119773954153061, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.071995735168457, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8559321761131287, "metrics/particle/momentum/class_4/event/energy_bias": -1.6779056787490845, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0555105209350586, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.144674777984619, "metrics/particle/momentum/class_4/event/px_bias": -0.09637249261140823, "metrics/particle/momentum/class_4/event/py_bias": 0.3240286707878113, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.782127022743225, "metrics/particle/momentum/class_4/event/pz_bias": 0.10672634840011597, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.125137329101562, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009573634713888168, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026416894048452377, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024991296231746674, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.011877105571329594, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05070878937840462, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18635901808738708, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.02762673981487751, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4777301549911499, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.044659800827503204, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05965915322303772, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.057429540902376175, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013636086136102676, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1672174632549286, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.291359007358551, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.11118184030056, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0754945278167725, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": -0.41885727643966675, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0813586711883545, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8836546540260315, "metrics/particle/momentum/class_5/event/px_bias": 0.012994922697544098, "metrics/particle/momentum/class_5/event/py_bias": 0.013047050684690475, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4969228506088257, "metrics/particle/momentum/class_5/event/pz_bias": 0.0974341556429863, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.8885042667388916, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.2683749198913574, "step_valid_time": 2773.2376358509064, "step_total_time": 2774.5060110092163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050753759145268, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11056357181264964, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3058, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.951427633738103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.948005232177894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9497133497133498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05199476782210595, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9455201837873318, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9421190320470896, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9438165438165438, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.951427633738103, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006663665757078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12169179876481717, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9345917471466199, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362357080035181, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9354130052724078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06376429199648197, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9297629499561019, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9313984168865436, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9305799648506151, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9345917471466199, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004240309710845, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12546926731615415, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1196, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9142640364188164, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9198473282442748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9170471841704718, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08015267175572519, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9074355083459787, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9129770992366413, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9101978691019786, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9142640364188164} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_15000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..db0d3fbf56db7bf71b0fa3ffe05a0e691ae6a33d --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7104964001515726, "train": {"Classification_binary": 2.981519937515259, "Classification": 0.024507783353328705, "Regression_pt": 0.16386771202087402, "Regression_eta": 0.01895879954099655, "Regression_sin_phi": 0.008518317714333534, "Regression_cos_phi": 0.008253788575530052, "Regression_energy": 0.16719000041484833, "Total": 9.866373062133789, "Total_unweighted": 3.37281633913517}, "train_group": {}, "valid": {"Classification_binary": 3.1164064407348633, "Classification": 0.02196529010931651, "Regression_pt": 0.16864550113677979, "Regression_eta": 0.013470947742462158, "Regression_sin_phi": 0.00766387643913428, "Regression_cos_phi": 0.008194820334513983, "Regression_energy": 0.17491002877553305, "Total": 9.717276255289713, "Total_unweighted": 3.511256905272603, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.022012552246451378, "diagnostic/loss/cld_pf/Classification_binary": 3.109396457672119, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007813035510480404, "diagnostic/loss/cld_pf/Regression_energy": 0.1738886833190918, "diagnostic/loss/cld_pf/Regression_eta": 0.012492435052990913, "diagnostic/loss/cld_pf/Regression_pt": 0.16885772347450256, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00716291693970561, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0798584371805191, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21707507967948914, "diagnostic/regression/cld_pf/energy_residual_mean": 0.052739161998033524, "diagnostic/regression/cld_pf/energy_residual_rms": 0.46415719389915466, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.0508740209043026, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.21368375420570374, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07804115861654282, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4620395600795746, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9239526391029358, "metrics/particle/class_1/pid_accuracy": 0.9831362962722778, "metrics/particle/class_2/efficiency": 0.39922523498535156, "metrics/particle/class_2/pid_accuracy": 0.6583904027938843, "metrics/particle/class_3/efficiency": 0.7628780007362366, "metrics/particle/class_3/pid_accuracy": 0.980267345905304, "metrics/particle/class_4/efficiency": 0.8627648949623108, "metrics/particle/class_4/pid_accuracy": 0.2210526317358017, "metrics/particle/class_5/efficiency": 0.9514563083648682, "metrics/particle/class_5/pid_accuracy": 0.48775508999824524, "metrics/particle/count/bias_mean": 1.426143765449524, "metrics/particle/count/mae": 3.9137253761291504, "metrics/particle/count/prediction_mean": 71.99150085449219, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06600108742713928, "metrics/particle/event/energy_response_mean": 0.9992495179176331, "metrics/particle/event/met_abs_error": 7.127111434936523, "metrics/particle/event/scalar_pt_relative_abs_error": 0.055502183735370636, "metrics/particle/event/scalar_pt_response_mean": 1.0190253257751465, "metrics/particle/event/vector_pt_closure": 0.05915963277220726, "metrics/particle/matched/delta_eta_abs_mean": 0.012035350315272808, "metrics/particle/matched/delta_phi_abs_mean": 0.013462983071804047, "metrics/particle/matched/delta_r_mean": 0.020119940862059593, "metrics/particle/matched/energy_relative_abs_mean": 0.09554596245288849, "metrics/particle/matched/pid_accuracy": 0.9588484168052673, "metrics/particle/matched/pt_relative_abs_mean": 0.09032139927148819, "metrics/particle/matching/duplicate_fraction": 0.04294261336326599, "metrics/particle/matching/efficiency": 0.8129764199256897, "metrics/particle/matching/f1": 0.8048433661460876, "metrics/particle/matching/purity": 0.7968714833259583, "metrics/particle/matching/target_coverage_dr0p05": 0.7683879137039185, "metrics/particle/matching/target_coverage_dr0p10": 0.8524706959724426, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7002732157707214, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8540360331535339, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02239232137799263, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09145508706569672, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0903010293841362, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02140803076326847, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14342133700847626, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0059472182765603065, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4108193814754486, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05404562130570412, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.711424469947815, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8071.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.301937997341156, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1636658906936646, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.314420461654663, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.194600224494934, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1383010745048523, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.170426368713379, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017350316047668457, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8535305261611938, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04907335340976715, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.343228578567505, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9971.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7961921691894531, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.618586540222168, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.5750732421875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.163603782653809, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.623668372631073, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.29546189308166504, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.272356986999512, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7539563179016113, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.98015022277832, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.031149856746196747, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09702424705028534, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0982184037566185, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021402545273303986, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15145480632781982, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020270269364118576, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0286861658096313, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2252078652381897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.925551652908325, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1480.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8646124005317688, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.723648190498352, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 83060.203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 83060.0234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2994219660758972, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4230999.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08400771021842957, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1487812995910645, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3831365704536438, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.742240905761719, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2595.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.538377583026886, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.7240304946899414, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.25764274597168, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3463925123214722, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.2661780118942261, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1257389336824417, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.077774047851562, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.7380423545837402, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.9532527923584, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0183396115899086, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09859723597764969, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09170564264059067, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015425278805196285, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14731021225452423, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006141248741187155, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1282346099615097, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0064904456958174706, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5582985281944275, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19540.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17258203029632568, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1199796199798584, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9090297818183899, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7832705974578857, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10645261406898499, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 39.99964904785156, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004024589899927378, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.31470587849617004, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.013297280296683311, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.700610637664795, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22611.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8546559810638428, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.5293567180633545, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.959153175354004, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.464128017425537, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.0270731449127197, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.9068771600723267, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.167125225067139, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.1020539999008179, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.77487850189209, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020115716382861137, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09653569757938385, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09164727479219437, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017389224842190742, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14645686745643616, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003093740437179804, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25244519114494324, "metrics/particle/momentum/all/accepted/pz_bias": -0.030811356380581856, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3438583612442017, "metrics/particle/momentum/all/accepted/samples": 29091.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2602992057800293, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1768951416015625, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 6128.291015625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 6128.16259765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1297154426574707, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1149164.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013702134601771832, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6765060424804688, "metrics/particle/momentum/all/assigned/pz_bias": -0.05072105675935745, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2657535076141357, "metrics/particle/momentum/all/assigned/samples": 35177.0, "metrics/particle/momentum/all/efficiency": 0.8137570023536682, "metrics/particle/momentum/all/event/energy_bias": 1.4239124059677124, "metrics/particle/momentum/all/event/p3_error_mean": 18.332063674926758, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.56376838684082, "metrics/particle/momentum/all/event/px_bias": 1.916919469833374, "metrics/particle/momentum/all/event/py_bias": 0.7371541261672974, "metrics/particle/momentum/all/event/pz_abs_mean": 13.330070495605469, "metrics/particle/momentum/all/event/pz_bias": -3.3899450302124023, "metrics/particle/momentum/all/event/pz_residual_rms": 22.452545166015625, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010187419131398201, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05658653751015663, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04664428159594536, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.013733293861150742, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09879115968942642, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017073479248210788, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16396114230155945, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.013359298929572105, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.975828230381012, "metrics/particle/momentum/class_1/accepted/samples": 15814.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09449146687984467, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3518725633621216, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.44063639640808105, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.38292646408081055, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.056894440203905106, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.1243977546691895, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006593919359147549, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3604500889778137, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.024313006550073624, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2872512340545654, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9224756360054016, "metrics/particle/momentum/class_1/event/energy_bias": 7.877399444580078, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.625235557556152, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.39327621459961, "metrics/particle/momentum/class_1/event/px_bias": 1.2019805908203125, "metrics/particle/momentum/class_1/event/py_bias": -0.14432239532470703, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.131855010986328, "metrics/particle/momentum/class_1/event/pz_bias": -0.8405922651290894, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.875350952148438, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04302365332841873, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20412878692150116, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20410381257534027, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05887912958860397, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2418890744447708, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04260869696736336, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9245814085006714, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.40272560715675354, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.642093181610107, "metrics/particle/momentum/class_2/accepted/samples": 1150.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.995219349861145, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9085223078727722, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 80967.390625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 80966.8671875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40345606207847595, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4177414.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08527422696352005, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.475520610809326, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.31180843710899353, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.957313060760498, "metrics/particle/momentum/class_2/assigned/samples": 2662.0, "metrics/particle/momentum/class_2/efficiency": 0.39792388677597046, "metrics/particle/momentum/class_2/event/energy_bias": -11.507438659667969, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.216730117797852, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.378263473510742, "metrics/particle/momentum/class_2/event/px_bias": 0.1385151892900467, "metrics/particle/momentum/class_2/event/py_bias": 0.7173574566841125, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.78253173828125, "metrics/particle/momentum/class_2/event/pz_bias": -1.6767607927322388, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.9385986328125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03166534751653671, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14068284630775452, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14193226397037506, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02914455719292164, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18367642164230347, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007721345173195004, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19933007657527924, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.022581247612833977, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9815428256988525, "metrics/particle/momentum/class_3/accepted/samples": 11656.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3248226046562195, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.183840036392212, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.126310348510742, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9848273992538452, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16618409752845764, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 50.75920104980469, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009090296924114227, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5437928438186646, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.037637244910001755, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7829980850219727, "metrics/particle/momentum/class_3/assigned/samples": 14851.0, "metrics/particle/momentum/class_3/efficiency": 0.7673974633216858, "metrics/particle/momentum/class_3/event/energy_bias": 7.022932052612305, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.891181945800781, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.264992713928223, "metrics/particle/momentum/class_3/event/px_bias": 0.5271774530410767, "metrics/particle/momentum/class_3/event/py_bias": -0.14674825966358185, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.551050186157227, "metrics/particle/momentum/class_3/event/pz_bias": -1.0451542139053345, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.328839302062988, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013578887097537518, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1092851385474205, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11047433316707611, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04386904463171959, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.161576509475708, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016447369009256363, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6180378794670105, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.09570807963609695, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1772818565368652, "metrics/particle/momentum/class_4/accepted/samples": 304.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.15765850245952606, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.4344539642333984, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5339958667755127, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4685449600219727, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11299435049295425, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.38059139251709, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6965065002441406, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.05046854540705681, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2248685359954834, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8587570786476135, "metrics/particle/momentum/class_4/event/energy_bias": -1.494611382484436, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.1130638122558594, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.103163480758667, "metrics/particle/momentum/class_4/event/px_bias": -0.0323307141661644, "metrics/particle/momentum/class_4/event/py_bias": 0.2611110508441925, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8982428312301636, "metrics/particle/momentum/class_4/event/pz_bias": 0.01961776614189148, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.774288177490234, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008298924192786217, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03407418355345726, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03480355814099312, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02010459639132023, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08437003195285797, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15843051671981812, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.07291631400585175, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.3953550159931183, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04751173034310341, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06419743597507477, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06491819769144058, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.012006514705717564, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.191691055893898, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.266795814037323, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.02055731788277626, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0372629165649414, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.4743685722351074, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0938725471496582, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8477070927619934, "metrics/particle/momentum/class_5/event/px_bias": 0.08157697319984436, "metrics/particle/momentum/class_5/event/py_bias": 0.049756262451410294, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5634984970092773, "metrics/particle/momentum/class_5/event/pz_bias": 0.15294450521469116, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.3216300010681152, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.1353931427001953, "step_valid_time": 2946.3300275802612, "step_total_time": 2947.46542096138, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0121768834767517, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10194431895585954, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9458483754512635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9492753623188406, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9475587703435805, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050724637681159424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9392845421726288, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9426877470355731, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.940983067565346, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9458483754512635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0098367434189015, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11138760303836825, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2137, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2126, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9381035996488147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9401671799384074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.939134256207427, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05983282006159261, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9332748024582967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9353277606687197, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.934300153812349, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9381035996488147, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0142269035186968, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.123170063917873, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1211, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1206, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9188163884673748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9188163884673748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9188163884673748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08118361153262518, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9150227617602428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9150227617602428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9150227617602428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9188163884673748} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_16000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..d69bbca19cf9c9ff4d8f6703df8202b822a87e72 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.757862826828344, "train": {"Classification_binary": 2.946959972381592, "Classification": 0.02169511653482914, "Regression_pt": 0.15615032613277435, "Regression_eta": 0.011472457088530064, "Regression_sin_phi": 0.006902249529957771, "Regression_cos_phi": 0.00846769567579031, "Regression_energy": 0.16036343574523926, "Total": 9.086651802062988, "Total_unweighted": 3.3120112530887127}, "train_group": {}, "valid": {"Classification_binary": 3.085227648417155, "Classification": 0.021860053141911823, "Regression_pt": 0.16831684112548828, "Regression_eta": 0.013061119864384333, "Regression_sin_phi": 0.007750510548551877, "Regression_cos_phi": 0.008204424753785133, "Regression_energy": 0.17526604731877646, "Total": 9.672667821248373, "Total_unweighted": 3.4796866451700526, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.022111989557743073, "diagnostic/loss/cld_pf/Classification_binary": 3.0931453704833984, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00783055555075407, "diagnostic/loss/cld_pf/Regression_energy": 0.17370814085006714, "diagnostic/loss/cld_pf/Regression_eta": 0.012403592467308044, "diagnostic/loss/cld_pf/Regression_pt": 0.16815808415412903, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007339584641158581, "diagnostic/regression/cld_pf/energy_pred_mean": 0.06036283075809479, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21214614808559418, "diagnostic/regression/cld_pf/energy_residual_mean": 0.03324354812502861, "diagnostic/regression/cld_pf/energy_residual_rms": 0.45175883173942566, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.029167305678129196, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2070811539888382, "diagnostic/regression/cld_pf/pt_residual_mean": 0.056334447115659714, "diagnostic/regression/cld_pf/pt_residual_rms": 0.44667455554008484, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9252750277519226, "metrics/particle/class_1/pid_accuracy": 0.9821247458457947, "metrics/particle/class_2/efficiency": 0.41266947984695435, "metrics/particle/class_2/pid_accuracy": 0.6681391596794128, "metrics/particle/class_3/efficiency": 0.768086314201355, "metrics/particle/class_3/pid_accuracy": 0.9831192493438721, "metrics/particle/class_4/efficiency": 0.8647830486297607, "metrics/particle/class_4/pid_accuracy": 0.2403734028339386, "metrics/particle/class_5/efficiency": 0.9669902920722961, "metrics/particle/class_5/pid_accuracy": 0.5401606559753418, "metrics/particle/count/bias_mean": 0.6790849566459656, "metrics/particle/count/mae": 3.7771241664886475, "metrics/particle/count/prediction_mean": 71.24444580078125, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0630916953086853, "metrics/particle/event/energy_response_mean": 0.9704530835151672, "metrics/particle/event/met_abs_error": 6.014955043792725, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050534218549728394, "metrics/particle/event/scalar_pt_response_mean": 0.995345950126648, "metrics/particle/event/vector_pt_closure": 0.054034274071455, "metrics/particle/matched/delta_eta_abs_mean": 0.012483452446758747, "metrics/particle/matched/delta_phi_abs_mean": 0.011250810697674751, "metrics/particle/matched/delta_r_mean": 0.018826544284820557, "metrics/particle/matched/energy_relative_abs_mean": 0.09555011987686157, "metrics/particle/matched/pid_accuracy": 0.9599238038063049, "metrics/particle/matched/pt_relative_abs_mean": 0.0900125578045845, "metrics/particle/matching/duplicate_fraction": 0.03860408812761307, "metrics/particle/matching/efficiency": 0.8169962763786316, "metrics/particle/matching/f1": 0.8130838871002197, "metrics/particle/matching/purity": 0.8092088103294373, "metrics/particle/matching/target_coverage_dr0p05": 0.785180389881134, "metrics/particle/matching/target_coverage_dr0p10": 0.8531931638717651, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7040059566497803, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8554346561431885, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021284935995936394, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09127797931432724, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08902102708816528, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01004031952470541, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14150778949260712, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005434782709926367, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.40484726428985596, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04194354638457298, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7572885751724243, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8096.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3063095808029175, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0226858854293823, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1816823482513428, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.053217887878418, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1318126767873764, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.593992233276367, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016653209924697876, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.840528666973114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.010655340738594532, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.09394907951355, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9908.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7986583709716797, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3581467866897583, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.61031436920166, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.804896354675293, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2616459131240845, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.040254563093185425, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.428565979003906, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.22450681030750275, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.146085739135742, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030217992141842842, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09894660115242004, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0997786819934845, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.00597200496122241, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1509571075439453, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02445472590625286, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.041354775428772, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20349852740764618, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.8324930667877197, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1513.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8788154125213623, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4862760305404663, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 15964.73046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 15964.5341796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2979053556919098, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 810421.0625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08727695792913437, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.077641010284424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3629026710987091, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.714598655700684, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2578.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5503819584846497, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.508527755737305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.212976932525635, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.199032187461853, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01569001004099846, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0576004683971405, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.056077003479004, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.4253242015838623, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.053882598876953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01684798114001751, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09760456532239914, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09086593985557556, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011881336569786072, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14570794999599457, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006626905524171889, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12870320677757263, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.013552851043641567, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5743207931518555, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19617.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17163190245628357, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6922507882118225, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6059539914131165, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47787588834762573, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10284473747015, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.049570083618164, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003367600031197071, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2806686460971832, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.003023109631612897, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2622607946395874, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22568.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8580238819122314, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.215421661734581, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.440299034118652, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.0236968994140625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.08943770825862885, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.4172499179840088, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.9522926807403564, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.03663848340511322, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.0681257247924805, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01876922696828842, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09592148661613464, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09081628918647766, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011065429076552391, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14483524858951569, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032163141295313835, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25244590640068054, "metrics/particle/momentum/all/accepted/pz_bias": -0.03125075250864029, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.355439305305481, "metrics/particle/momentum/all/accepted/samples": 29226.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2617073953151703, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8440436720848083, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1174.8289794921875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1174.6959228515625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12537798285484314, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 219777.578125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013293775729835033, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6446124911308289, "metrics/particle/momentum/all/assigned/pz_bias": -0.031647223979234695, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.052321434020996, "metrics/particle/momentum/all/assigned/samples": 35054.0, "metrics/particle/momentum/all/efficiency": 0.8175333738327026, "metrics/particle/momentum/all/event/energy_bias": -7.082096576690674, "metrics/particle/momentum/all/event/p3_error_mean": 16.287975311279297, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.637757301330566, "metrics/particle/momentum/all/event/px_bias": -0.3667736053466797, "metrics/particle/momentum/all/event/py_bias": 0.43459582328796387, "metrics/particle/momentum/all/event/pz_abs_mean": 11.690573692321777, "metrics/particle/momentum/all/event/pz_bias": -2.686469316482544, "metrics/particle/momentum/all/event/pz_residual_rms": 18.926685333251953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0108775794506073, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05540940910577774, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04568866267800331, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009515507146716118, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09634747356176376, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018939394503831863, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15942761301994324, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.014591123908758163, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0329492092132568, "metrics/particle/momentum/class_1/accepted/samples": 15840.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10529744625091553, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2524804174900055, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2946314215660095, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.22960825264453888, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.055312443524599075, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.3588857650756836, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006826535798609257, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.35686907172203064, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01514489483088255, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.315574884414673, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9239922761917114, "metrics/particle/momentum/class_1/event/energy_bias": -0.2677222788333893, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.421490669250488, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.690803050994873, "metrics/particle/momentum/class_1/event/px_bias": 0.23565945029258728, "metrics/particle/momentum/class_1/event/py_bias": -0.02489054575562477, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.148409843444824, "metrics/particle/momentum/class_1/event/pz_bias": -0.9145815372467041, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.07511329650879, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04121406376361847, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20824994146823883, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20820730924606323, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.059885282069444656, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24632474780082703, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.042904291301965714, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9362150430679321, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1308363825082779, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.7509894371032715, "metrics/particle/momentum/class_2/accepted/samples": 1212.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0025888681411743, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7461336255073547, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 15617.1884765625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 15616.673828125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.39620494842529297, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 801607.6875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0865275114774704, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.3504433631896973, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.12200185656547546, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.739976406097412, "metrics/particle/momentum/class_2/assigned/samples": 2635.0, "metrics/particle/momentum/class_2/efficiency": 0.4193771481513977, "metrics/particle/momentum/class_2/event/energy_bias": -12.413839340209961, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.140823364257812, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.452757835388184, "metrics/particle/momentum/class_2/event/px_bias": -0.1517709493637085, "metrics/particle/momentum/class_2/event/py_bias": 0.7438671588897705, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.692851066589355, "metrics/particle/momentum/class_2/event/pz_bias": -1.3350038528442383, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.575458526611328, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02732206881046295, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13951386511325836, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13987155258655548, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019465530291199684, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1808367669582367, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007690992788411677, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19573861360549927, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.047307442873716354, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8272918462753296, "metrics/particle/momentum/class_3/accepted/samples": 11702.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31527790427207947, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5166205167770386, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7089389562606812, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5620739459991455, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15983189642429352, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.326763153076172, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007862807251513004, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.49782896041870117, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03793181851506233, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.248138427734375, "metrics/particle/momentum/class_3/assigned/samples": 14753.0, "metrics/particle/momentum/class_3/efficiency": 0.7704259753227234, "metrics/particle/momentum/class_3/event/energy_bias": 7.2945027351379395, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.659286499023438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.375371932983398, "metrics/particle/momentum/class_3/event/px_bias": -0.3237730860710144, "metrics/particle/momentum/class_3/event/py_bias": -0.5826195478439331, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.229214191436768, "metrics/particle/momentum/class_3/event/pz_bias": -0.8861042261123657, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.678326606750488, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01588757522404194, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11793810874223709, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12001226097345352, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05033831670880318, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.171639546751976, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009836065582931042, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5898827910423279, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08735300600528717, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1148841381073, "metrics/particle/momentum/class_4/accepted/samples": 305.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.19414864480495453, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.556159496307373, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.5208888053894043, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.4466049671173096, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11299435049295425, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.388795852661133, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.713115394115448, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10489089787006378, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.4260618686676025, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8615819215774536, "metrics/particle/momentum/class_4/event/energy_bias": -1.600462555885315, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.7960524559020996, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9221075773239136, "metrics/particle/momentum/class_4/event/px_bias": -0.22918125987052917, "metrics/particle/momentum/class_4/event/py_bias": 0.22800861299037933, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6586482524871826, "metrics/particle/momentum/class_4/event/pz_bias": 0.2521466016769409, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.581748962402344, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010351105593144894, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028473034501075745, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028504930436611176, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012662953697144985, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07085514068603516, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2126360982656479, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.019829943776130676, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.614786684513092, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04249941185116768, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.08206722885370255, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.07168431580066681, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03537522628903389, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3676091134548187, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.3152366280555725, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.03376759588718414, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0379589796066284, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.09457550197839737, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1525384187698364, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8915265202522278, "metrics/particle/momentum/class_5/event/px_bias": 0.1022922545671463, "metrics/particle/momentum/class_5/event/py_bias": 0.07023017108440399, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5813581943511963, "metrics/particle/momentum/class_5/event/pz_bias": 0.19707359373569489, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.3707478046417236, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.303168535232544, "step_valid_time": 3115.656457424164, "step_total_time": 3116.959626674652, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017422930612643, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10411337023377132, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2884, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2870, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.946504758779127, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9508737223870755, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9486842105263158, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0491262776129245, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9419100754840827, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9462578305308276, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.944078947368421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.946504758779127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014246306544559, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10990281331503171, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2129, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9411764705882353, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.943246810382754, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.942210503186113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05675318961724593, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9345917471466199, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9366476022877255, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.935618545374643, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9411764705882353, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015282874239073, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11401164278734377, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1204, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9165402124430956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9172361427486713, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.916888045540797, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08276385725132877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.91350531107739, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9141989369779803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9138519924098671, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9165402124430956} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_17000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..b6b6945189357ae5aa6f710f4f039d5fd5328d90 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8052292535051155, "train": {"Classification_binary": 3.052511692047119, "Classification": 0.02221434749662876, "Regression_pt": 0.16875627636909485, "Regression_eta": 0.016344092786312103, "Regression_sin_phi": 0.00662841135635972, "Regression_cos_phi": 0.006905410438776016, "Regression_energy": 0.18034067749977112, "Total": 9.669906616210938, "Total_unweighted": 3.4537009079940617}, "train_group": {}, "valid": {"Classification_binary": 3.044027328491211, "Classification": 0.021472424268722534, "Regression_pt": 0.1668498714764913, "Regression_eta": 0.012909039855003357, "Regression_sin_phi": 0.0074490737169981, "Regression_cos_phi": 0.007902001962065697, "Regression_energy": 0.17319426933924356, "Total": 9.521857579549154, "Total_unweighted": 3.4338040091097355, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.02156347781419754, "diagnostic/loss/cld_pf/Classification_binary": 3.035677909851074, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007554471492767334, "diagnostic/loss/cld_pf/Regression_energy": 0.17208795249462128, "diagnostic/loss/cld_pf/Regression_eta": 0.012063097208738327, "diagnostic/loss/cld_pf/Regression_pt": 0.16700482368469238, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006967488676309586, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08254582434892654, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21605899930000305, "diagnostic/regression/cld_pf/energy_residual_mean": 0.055426549166440964, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4610436260700226, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.044050563126802444, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2103564590215683, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07121770083904266, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4544444978237152, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9230327010154724, "metrics/particle/class_1/pid_accuracy": 0.9823927283287048, "metrics/particle/class_2/efficiency": 0.3937564194202423, "metrics/particle/class_2/pid_accuracy": 0.7022569179534912, "metrics/particle/class_3/efficiency": 0.754329264163971, "metrics/particle/class_3/pid_accuracy": 0.9798688888549805, "metrics/particle/class_4/efficiency": 0.8587285280227661, "metrics/particle/class_4/pid_accuracy": 0.2538190484046936, "metrics/particle/class_5/efficiency": 0.9650485515594482, "metrics/particle/class_5/pid_accuracy": 0.5573440790176392, "metrics/particle/count/bias_mean": -0.7980391979217529, "metrics/particle/count/mae": 3.7274510860443115, "metrics/particle/count/prediction_mean": 69.76731872558594, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06441421806812286, "metrics/particle/event/energy_response_mean": 0.9777292013168335, "metrics/particle/event/met_abs_error": 6.805209636688232, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0532197505235672, "metrics/particle/event/scalar_pt_response_mean": 0.9955056309700012, "metrics/particle/event/vector_pt_closure": 0.05670947581529617, "metrics/particle/matched/delta_eta_abs_mean": 0.01047125831246376, "metrics/particle/matched/delta_phi_abs_mean": 0.011538710445165634, "metrics/particle/matched/delta_r_mean": 0.01745244674384594, "metrics/particle/matched/energy_relative_abs_mean": 0.09544018656015396, "metrics/particle/matched/pid_accuracy": 0.9607863426208496, "metrics/particle/matched/pt_relative_abs_mean": 0.0894445851445198, "metrics/particle/matching/duplicate_fraction": 0.03680768981575966, "metrics/particle/matching/efficiency": 0.8085120320320129, "metrics/particle/matching/f1": 0.813109815120697, "metrics/particle/matching/purity": 0.8177602291107178, "metrics/particle/matching/target_coverage_dr0p05": 0.7837169170379639, "metrics/particle/matching/target_coverage_dr0p10": 0.845718502998352, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6986245512962341, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8457092642784119, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02003689669072628, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09221179038286209, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08965291827917099, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019897030666470528, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14237254858016968, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007154512219130993, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.42594343423843384, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.014478275552392006, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.778399109840393, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7967.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29970765113830566, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.257766604423523, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.350409746170044, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2284166812896729, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13666121661663055, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.963865280151367, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020458264276385307, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8963197469711304, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.020901815965771675, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2548112869262695, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9776.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.785932719707489, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0418039560317993, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.950108528137207, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.9171829223632812, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.14339226484298706, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.08010309934616089, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.755002975463867, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.08655983209609985, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.302763938903809, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028039677068591118, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10026726871728897, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10218413174152374, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.020978856831789017, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1535969227552414, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021088436245918274, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9313531517982483, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06633692979812622, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.987079381942749, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1470.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8512095212936401, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6285501718521118, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 33958.62109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 33958.44140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2952534258365631, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1692993.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08205953240394592, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1170578002929688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5114520788192749, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.11776351928711, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2486.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5347399115562439, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.6133477687835693, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.306802749633789, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.263748049736023, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.08422593772411346, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10066713392734528, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.137545585632324, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.7324110269546509, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.359024047851562, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015338125638663769, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09702154994010925, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08954619616270065, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01160343736410141, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14469289779663086, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006161745986901224, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12485476583242416, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0003645518736448139, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5532841682434082, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19475.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1698249727487564, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9830288290977478, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8021951913833618, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6738728284835815, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1046568751335144, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 29.81397247314453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003661204595118761, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3003581464290619, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.015707237645983696, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4741772413253784, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22397.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8518129587173462, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.33733147382736206, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.73847770690918, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.300851821899414, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6167014837265015, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.26183003187179565, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.96622371673584, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8744648694992065, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.947534084320068, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01727871596813202, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09586119651794434, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09021817147731781, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01436550635844469, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1445251852273941, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003458771388977766, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24882842600345612, "metrics/particle/momentum/all/accepted/pz_bias": 0.0008623679750598967, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.237489938735962, "metrics/particle/momentum/all/accepted/samples": 28912.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25533393025398254, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1068235635757446, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2436.66259765625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2436.532958984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12735508382320404, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 453417.15625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014022331684827805, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.670491099357605, "metrics/particle/momentum/all/assigned/pz_bias": -0.020639335736632347, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.217982530593872, "metrics/particle/momentum/all/assigned/samples": 34659.0, "metrics/particle/momentum/all/efficiency": 0.8087499141693115, "metrics/particle/momentum/all/event/energy_bias": -4.992483139038086, "metrics/particle/momentum/all/event/p3_error_mean": 17.37004280090332, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.277358055114746, "metrics/particle/momentum/all/event/px_bias": 0.8443197011947632, "metrics/particle/momentum/all/event/py_bias": -0.24126602709293365, "metrics/particle/momentum/all/event/pz_abs_mean": 12.449357032775879, "metrics/particle/momentum/all/event/pz_bias": -0.7713862657546997, "metrics/particle/momentum/all/event/pz_residual_rms": 20.84514808654785, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010429788380861282, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05481816455721855, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04523041844367981, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011842573061585426, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09595903754234314, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002216312102973461, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15514710545539856, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0004523445386439562, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8568450212478638, "metrics/particle/momentum/class_1/accepted/samples": 15792.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10648277401924133, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3143381178379059, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4005201458930969, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.33577436208724976, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0592285692691803, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.021775245666504, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008052751421928406, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.38604500889778137, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.012668727897107601, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4365053176879883, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9211923480033875, "metrics/particle/momentum/class_1/event/energy_bias": 2.31999135017395, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.353296279907227, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.806235313415527, "metrics/particle/momentum/class_1/event/px_bias": 0.47566479444503784, "metrics/particle/momentum/class_1/event/py_bias": -0.3728277385234833, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.9452543258667, "metrics/particle/momentum/class_1/event/pz_bias": -0.6811579465866089, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.72210693359375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03861100226640701, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20694269239902496, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20994499325752258, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06950881332159042, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24695470929145813, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04694419726729393, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0378592014312744, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.32400357723236084, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.829161167144775, "metrics/particle/momentum/class_2/accepted/samples": 1129.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9561527371406555, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8962939381599426, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 33833.90234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 33833.38671875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3955911695957184, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1689937.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08296593278646469, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4185526371002197, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.27214476466178894, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.954641819000244, "metrics/particle/momentum/class_2/assigned/samples": 2495.0, "metrics/particle/momentum/class_2/efficiency": 0.3906574249267578, "metrics/particle/momentum/class_2/event/energy_bias": -10.859338760375977, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.505090713500977, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.572579383850098, "metrics/particle/momentum/class_2/event/px_bias": -0.5288229584693909, "metrics/particle/momentum/class_2/event/py_bias": -0.18187034130096436, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.913585662841797, "metrics/particle/momentum/class_2/event/pz_bias": -2.2216296195983887, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.353607177734375, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.024764681234955788, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14171770215034485, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14043524861335754, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0250308308750391, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18184976279735565, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007812500116415322, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19367307424545288, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.028219593688845634, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6522850394248962, "metrics/particle/momentum/class_3/accepted/samples": 11520.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31481459736824036, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0955958366394043, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0349173545837402, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.892149567604065, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1631724089384079, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 39.156368255615234, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00937931053340435, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5388697981834412, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.008229522034525871, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5886292457580566, "metrics/particle/momentum/class_3/assigned/samples": 14500.0, "metrics/particle/momentum/class_3/efficiency": 0.7584435939788818, "metrics/particle/momentum/class_3/event/energy_bias": 4.81756067276001, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.778661727905273, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.340781211853027, "metrics/particle/momentum/class_3/event/px_bias": 0.6665141582489014, "metrics/particle/momentum/class_3/event/py_bias": 0.10529614239931107, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.400519371032715, "metrics/particle/momentum/class_3/event/pz_bias": 1.419993281364441, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.860394477844238, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014815228059887886, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11296596378087997, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11270391196012497, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.050651293247938156, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15877868235111237, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009900989942252636, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6108551025390625, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.15970714390277863, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.16522479057312, "metrics/particle/momentum/class_4/accepted/samples": 303.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.19491751492023468, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.9697027802467346, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.941340446472168, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8703851103782654, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11864406615495682, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.090927124023438, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6681942343711853, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.18765968084335327, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.3626668453216553, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8559321761131287, "metrics/particle/momentum/class_4/event/energy_bias": -1.1184415817260742, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.83064341545105, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.015155076980591, "metrics/particle/momentum/class_4/event/px_bias": 0.04039178788661957, "metrics/particle/momentum/class_4/event/py_bias": 0.2319183349609375, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6604130268096924, "metrics/particle/momentum/class_4/event/pz_bias": 0.6023060083389282, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.550936698913574, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008839704096317291, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03211677819490433, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.030462130904197693, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.018396075814962387, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07844489812850952, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16129887104034424, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.06016909331083298, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.42177048325538635, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.031276214867830276, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05154823139309883, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05013911426067352, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.018300173804163933, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14730098843574524, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2511773407459259, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.02886216528713703, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9567669630050659, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": -0.1522558480501175, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9822883605957031, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8092154860496521, "metrics/particle/momentum/class_5/event/px_bias": 0.1905718892812729, "metrics/particle/momentum/class_5/event/py_bias": -0.023782435804605484, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4431811273097992, "metrics/particle/momentum/class_5/event/pz_bias": 0.10910186916589737, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7490332126617432, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.323908805847168, "step_valid_time": 3284.342646598816, "step_total_time": 3285.6665556430817, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9966188573332628, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09944946957112033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.951427633738103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9517399868680236, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9515837846709339, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048260013131976365, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9458483754512635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9461588969139856, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9460036107008042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.951427633738103, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9967374066251242, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10918077776160795, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2266, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2134, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9416154521510096, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9466019417475728, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9441021126760564, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05339805825242718, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9367866549604916, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.941747572815534, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9392605633802817, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9416154521510096, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988884615073876, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11361299514566614, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1202, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9165402124430956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9193302891933028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9179331306990881, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0806697108066971, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9119878603945372, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9147640791476408, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9133738601823708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9165402124430956} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_18000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..553ee24e6950b1e13ca689feed30f4c77c296fb9 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.852595680181887, "train": {"Classification_binary": 3.0707855224609375, "Classification": 0.021730396896600723, "Regression_pt": 0.1821591556072235, "Regression_eta": 0.01694977469742298, "Regression_sin_phi": 0.009012882597744465, "Regression_cos_phi": 0.007344000041484833, "Regression_energy": 0.1846618950366974, "Total": 10.157611846923828, "Total_unweighted": 3.4926436273381114}, "train_group": {}, "valid": {"Classification_binary": 3.0178352991739907, "Classification": 0.021114786465962727, "Regression_pt": 0.1642408569653829, "Regression_eta": 0.012863285839557648, "Regression_sin_phi": 0.007406134779254596, "Regression_cos_phi": 0.007907085741559664, "Regression_energy": 0.17012566328048706, "Total": 9.411880493164062, "Total_unweighted": 3.4014931122461953, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.021237913519144058, "diagnostic/loss/cld_pf/Classification_binary": 3.0352821350097656, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007383388001471758, "diagnostic/loss/cld_pf/Regression_energy": 0.16877704858779907, "diagnostic/loss/cld_pf/Regression_eta": 0.011959388852119446, "diagnostic/loss/cld_pf/Regression_pt": 0.16480405628681183, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006979185622185469, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07989225536584854, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2164442390203476, "diagnostic/regression/cld_pf/energy_residual_mean": 0.052772972732782364, "diagnostic/regression/cld_pf/energy_residual_rms": 0.46033138036727905, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04288686066865921, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.21164870262145996, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07005400210618973, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4570438861846924, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9239143133163452, "metrics/particle/class_1/pid_accuracy": 0.9813931584358215, "metrics/particle/class_2/efficiency": 0.41688504815101624, "metrics/particle/class_2/pid_accuracy": 0.7070237994194031, "metrics/particle/class_3/efficiency": 0.763757050037384, "metrics/particle/class_3/pid_accuracy": 0.9802612662315369, "metrics/particle/class_4/efficiency": 0.8335015177726746, "metrics/particle/class_4/pid_accuracy": 0.2615012228488922, "metrics/particle/class_5/efficiency": 0.9592233300209045, "metrics/particle/class_5/pid_accuracy": 0.568825900554657, "metrics/particle/count/bias_mean": 0.3261438012123108, "metrics/particle/count/mae": 3.616339921951294, "metrics/particle/count/prediction_mean": 70.8915023803711, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06356513500213623, "metrics/particle/event/energy_response_mean": 1.0044240951538086, "metrics/particle/event/met_abs_error": 6.979982376098633, "metrics/particle/event/scalar_pt_relative_abs_error": 0.052560433745384216, "metrics/particle/event/scalar_pt_response_mean": 1.0142837762832642, "metrics/particle/event/vector_pt_closure": 0.05796750634908676, "metrics/particle/matched/delta_eta_abs_mean": 0.012223979458212852, "metrics/particle/matched/delta_phi_abs_mean": 0.012310867197811604, "metrics/particle/matched/delta_r_mean": 0.0193411223590374, "metrics/particle/matched/energy_relative_abs_mean": 0.09642843157052994, "metrics/particle/matched/pid_accuracy": 0.9604507684707642, "metrics/particle/matched/pt_relative_abs_mean": 0.0896909162402153, "metrics/particle/matching/duplicate_fraction": 0.039340242743492126, "metrics/particle/matching/efficiency": 0.814532458782196, "metrics/particle/matching/f1": 0.8126544952392578, "metrics/particle/matching/purity": 0.810785174369812, "metrics/particle/matching/target_coverage_dr0p05": 0.7783726453781128, "metrics/particle/matching/target_coverage_dr0p10": 0.8526281714439392, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7010976076126099, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8521558046340942, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020658759400248528, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09148276597261429, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08874688297510147, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009645792655646801, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1415606290102005, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005706488154828548, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4158574640750885, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.026770785450935364, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6979992389678955, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8061.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3087686598300934, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2637832164764404, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.357763409614563, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2291252613067627, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13875404000282288, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.3816556930542, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019215211272239685, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.885397732257843, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.017066236585378647, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.186190128326416, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9888.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7952056527137756, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.0640275478363037, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.430330276489258, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.273667335510254, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.43852707743644714, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.09656745195388794, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.053064346313477, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0757529735565186, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.49302864074707, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030523257330060005, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09977972507476807, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10028340667486191, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006771862041205168, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15330849587917328, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024967148900032043, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0615828037261963, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.32609325647354126, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.453650712966919, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1522.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8976948857307434, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.031748056411743, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 19918.701171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 19918.51171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3006230592727661, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1009187.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09306853264570236, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.3417859077453613, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.46063631772994995, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.076525688171387, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2568.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5536558628082275, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0809619426727295, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.496341228485107, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.240767240524292, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03353188931941986, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.01704120822250843, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.338136672973633, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2231197357177734, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.70075798034668, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01775665022432804, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09991075843572617, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0915641337633133, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009379752911627293, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14768244326114655, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004602638946380466, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12644480168819427, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.014095887541770935, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.505084216594696, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19554.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17269229888916016, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7663534283638, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6107829809188843, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.48245948553085327, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10541500151157379, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.103926181793213, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003994673956185579, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.30945897102355957, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013404219411313534, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.592898964881897, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22530.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8552683591842651, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.48481208086013794, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.955010414123535, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.360696792602539, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.9415096044540405, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.1227579116821289, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.250877857208252, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.590904712677002, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.53248119354248, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019226420670747757, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09757222980260849, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09124017506837845, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009317129850387573, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14631706476211548, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00319181801751256, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2553611695766449, "metrics/particle/momentum/all/accepted/pz_bias": -0.014980343170464039, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2617144584655762, "metrics/particle/momentum/all/accepted/samples": 29137.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2643669545650482, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9998216032981873, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1462.8253173828125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1462.6927490234375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12916594743728638, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 273414.875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014834505505859852, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6948102116584778, "metrics/particle/momentum/all/assigned/pz_bias": -0.020355742424726486, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4577441215515137, "metrics/particle/momentum/all/assigned/samples": 34986.0, "metrics/particle/momentum/all/efficiency": 0.8150438070297241, "metrics/particle/momentum/all/event/energy_bias": 2.4678778648376465, "metrics/particle/momentum/all/event/p3_error_mean": 18.449872970581055, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.610572814941406, "metrics/particle/momentum/all/event/px_bias": 2.4135684967041016, "metrics/particle/momentum/all/event/py_bias": -0.20228414237499237, "metrics/particle/momentum/all/event/pz_abs_mean": 13.611011505126953, "metrics/particle/momentum/all/event/pz_bias": 0.4435378909111023, "metrics/particle/momentum/all/event/pz_residual_rms": 22.45233726501465, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01061695534735918, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05709335580468178, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04598872736096382, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00786755420267582, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09882766753435135, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015801782719790936, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16260527074337006, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0015164004871621728, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9138044714927673, "metrics/particle/momentum/class_1/accepted/samples": 15821.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10725502669811249, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.34428146481513977, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4063504636287689, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.339205801486969, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.057598039507865906, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.535400867462158, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006711017806082964, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3782859146595001, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01366321463137865, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.567551851272583, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9228839874267578, "metrics/particle/momentum/class_1/event/energy_bias": 2.7420730590820312, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.572229385375977, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.855960845947266, "metrics/particle/momentum/class_1/event/px_bias": 0.3673606514930725, "metrics/particle/momentum/class_1/event/py_bias": 0.2034216672182083, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.204039573669434, "metrics/particle/momentum/class_1/event/pz_bias": 0.15199901163578033, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.9178466796875, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03956791013479233, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19905762374401093, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20040304958820343, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.051141366362571716, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23839795589447021, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04466501250863075, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.901418685913086, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3688824474811554, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.505356788635254, "metrics/particle/momentum/class_2/accepted/samples": 1209.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9787559509277344, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9005151987075806, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 19446.541015625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 19446.052734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4007604420185089, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 997221.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09239543974399567, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.449824333190918, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.21175804734230042, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.930568218231201, "metrics/particle/momentum/class_2/assigned/samples": 2630.0, "metrics/particle/momentum/class_2/efficiency": 0.4183391034603119, "metrics/particle/momentum/class_2/event/energy_bias": -4.675661563873291, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.623875617980957, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.615997314453125, "metrics/particle/momentum/class_2/event/px_bias": 1.2831908464431763, "metrics/particle/momentum/class_2/event/py_bias": -0.16160723567008972, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.062933921813965, "metrics/particle/momentum/class_2/event/pz_bias": -0.5844150185585022, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.528146743774414, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029050104320049286, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14267268776893616, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1417880654335022, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016828889027237892, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18351683020591736, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008582217851653695, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20181335508823395, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.005192272365093231, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7834588885307312, "metrics/particle/momentum/class_3/accepted/samples": 11652.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3231150507926941, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7693532705307007, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.79924476146698, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6506890058517456, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16450010240077972, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.258442878723145, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010549240745604038, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5757487416267395, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.027046414092183113, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0656559467315674, "metrics/particle/momentum/class_3/assigned/samples": 14693.0, "metrics/particle/momentum/class_3/efficiency": 0.7671341300010681, "metrics/particle/momentum/class_3/event/energy_bias": 6.3200602531433105, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.582052230834961, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.206562042236328, "metrics/particle/momentum/class_3/event/px_bias": 0.6467685103416443, "metrics/particle/momentum/class_3/event/py_bias": -0.3744756877422333, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.274167537689209, "metrics/particle/momentum/class_3/event/pz_bias": 0.33218762278556824, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.343513488769531, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014451980590820312, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10872529447078705, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10824236273765564, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04111116752028465, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15503409504890442, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013888888992369175, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6623168587684631, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12632110714912415, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2454285621643066, "metrics/particle/momentum/class_4/accepted/samples": 288.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2387492060661316, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9750300645828247, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.669175624847412, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6080198287963867, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.32012414932251, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7101713418960571, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.01734760031104088, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1809229850769043, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8135592937469482, "metrics/particle/momentum/class_4/event/energy_bias": -1.7040327787399292, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5937695503234863, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8393341302871704, "metrics/particle/momentum/class_4/event/px_bias": -0.023360438644886017, "metrics/particle/momentum/class_4/event/py_bias": 0.2054397463798523, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4849205017089844, "metrics/particle/momentum/class_4/event/pz_bias": 0.4891715943813324, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.515811443328857, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010405606590211391, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03169218823313713, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03174758702516556, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005390897858887911, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08503067493438721, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16040995717048645, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.057637929916381836, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4093702733516693, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.029168955981731415, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.08976148813962936, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.09560418874025345, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05420453101396561, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5248896479606628, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.24511517584323883, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.01084813941270113, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9098172187805176, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.21456149220466614, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9755792021751404, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7933086156845093, "metrics/particle/momentum/class_5/event/px_bias": 0.13960911333560944, "metrics/particle/momentum/class_5/event/py_bias": -0.07506264001131058, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.45596843957901, "metrics/particle/momentum/class_5/event/pz_bias": 0.054594606161117554, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7513561248779297, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.1921110153198242, "step_valid_time": 3448.7220692634583, "step_total_time": 3449.9141805171967, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987928141082574, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09700905990385644, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.951427633738103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9536184210526316, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9525217677016593, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04638157894736842, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9471611421069904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9493421052631579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9482503696402168, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.951427633738103, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0034691360058712, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10833655216286053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9520668425681619, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9512302284710018, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04793315743183817, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9424934152765584, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.94415127528584, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9433216168717048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9503950834064969, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007577248321321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11526367463268272, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1219, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1212, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.924886191198786, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9269961977186312, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9259399924041017, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07300380228136882, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9195751138088012, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9216730038022813, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.920622863653627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.924886191198786} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_19000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..db761dc320c40f030e4102e787f859473f537138 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.8999621068586586, "train": {"Classification_binary": 2.985032081604004, "Classification": 0.021517902612686157, "Regression_pt": 0.14755840599536896, "Regression_eta": 0.01135941594839096, "Regression_sin_phi": 0.006472564768046141, "Regression_cos_phi": 0.007870317436754704, "Regression_energy": 0.1527019888162613, "Total": 8.847726821899414, "Total_unweighted": 3.332512677181512}, "train_group": {}, "valid": {"Classification_binary": 2.9764540990193686, "Classification": 0.02095278600851695, "Regression_pt": 0.16190333167711893, "Regression_eta": 0.012769830723603567, "Regression_sin_phi": 0.007448742787043254, "Regression_cos_phi": 0.007902564480900764, "Regression_energy": 0.16746187210083008, "Total": 9.307075500488281, "Total_unweighted": 3.3548932267973823, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.021056082099676132, "diagnostic/loss/cld_pf/Classification_binary": 2.982205867767334, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00751287117600441, "diagnostic/loss/cld_pf/Regression_energy": 0.1671368032693863, "diagnostic/loss/cld_pf/Regression_eta": 0.012293213978409767, "diagnostic/loss/cld_pf/Regression_pt": 0.1628958135843277, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0069024283438920975, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07676786184310913, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21219156682491302, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04964858293533325, "diagnostic/regression/cld_pf/energy_residual_rms": 0.45405417680740356, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03310110792517662, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20573389530181885, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06026824563741684, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4456506669521332, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9271723628044128, "metrics/particle/class_1/pid_accuracy": 0.9830501675605774, "metrics/particle/class_2/efficiency": 0.40423834323883057, "metrics/particle/class_2/pid_accuracy": 0.6840473413467407, "metrics/particle/class_3/efficiency": 0.7632516026496887, "metrics/particle/class_3/pid_accuracy": 0.9828395247459412, "metrics/particle/class_4/efficiency": 0.8486377596855164, "metrics/particle/class_4/pid_accuracy": 0.28894174098968506, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.5578093528747559, "metrics/particle/count/bias_mean": -0.23594771325588226, "metrics/particle/count/mae": 3.6607842445373535, "metrics/particle/count/prediction_mean": 70.32941436767578, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06285365670919418, "metrics/particle/event/energy_response_mean": 0.9779653549194336, "metrics/particle/event/met_abs_error": 6.163529396057129, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051301345229148865, "metrics/particle/event/scalar_pt_response_mean": 0.9890506863594055, "metrics/particle/event/vector_pt_closure": 0.054841503500938416, "metrics/particle/matched/delta_eta_abs_mean": 0.01238777581602335, "metrics/particle/matched/delta_phi_abs_mean": 0.011314195580780506, "metrics/particle/matched/delta_r_mean": 0.018732963129878044, "metrics/particle/matched/energy_relative_abs_mean": 0.09511931240558624, "metrics/particle/matched/pid_accuracy": 0.9618938565254211, "metrics/particle/matched/pt_relative_abs_mean": 0.0885448232293129, "metrics/particle/matching/duplicate_fraction": 0.0354633666574955, "metrics/particle/matching/efficiency": 0.8149955868721008, "metrics/particle/matching/f1": 0.8163604140281677, "metrics/particle/matching/purity": 0.8177298307418823, "metrics/particle/matching/target_coverage_dr0p05": 0.7788450121879578, "metrics/particle/matching/target_coverage_dr0p10": 0.8512295484542847, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7026165723800659, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8522206544876099, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020824698731303215, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0918179526925087, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08786731958389282, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011399460956454277, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1413567215204239, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004122938495129347, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.39718204736709595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03184836730360985, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5580824613571167, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8004.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2992279827594757, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0104384422302246, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0724451541900635, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9460838437080383, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13641004264354706, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.32395076751709, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016336532309651375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8507463932037354, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02716369740664959, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.360339879989624, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9794.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7895827293395996, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.24827462434768677, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.71934700012207, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8227968215942383, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.13542892038822174, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07555738091468811, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.571542263031006, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.27127018570899963, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.467238426208496, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029531164094805717, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10190653055906296, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10410035401582718, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0024681708309799433, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1570807248353958, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02169625274837017, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9868381023406982, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.22732537984848022, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1444146633148193, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1521.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8231688141822815, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.516966462135315, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 80592.28125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 80592.078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28127521276474, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4011686.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0786924958229065, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8730385303497314, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.11358516663312912, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.17520809173584, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2478.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5532920956611633, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.6401658058166504, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.213383197784424, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2001794576644897, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07252948731184006, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12358056008815765, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.035482406616211, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7469133734703064, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.910160064697266, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017137715592980385, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09727170318365097, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08943796157836914, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010075646452605724, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1453687995672226, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005093984073027968, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12575967609882355, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005337479989975691, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5156229734420776, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19631.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16919104754924774, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6724962592124939, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5581846237182617, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42982158064842224, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10050676017999649, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.740358829498291, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00368954474106431, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.28694769740104675, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005707681179046631, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.348587989807129, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22496.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.858636200428009, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.6280946731567383, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.327439308166504, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.972645282745361, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.8290016651153564, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8270711302757263, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.721041440963745, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.15238213539123535, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.64897346496582, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018796415999531746, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09601631760597229, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08977168798446655, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010042200796306133, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14491748809814453, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026066675782203674, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24519185721874237, "metrics/particle/momentum/all/accepted/pz_bias": -0.024195924401283264, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1667256355285645, "metrics/particle/momentum/all/accepted/samples": 29156.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2524324953556061, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8278806209564209, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5744.6708984375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5744.53857421875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12350437045097351, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1070995.75, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01259779091924429, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6300844550132751, "metrics/particle/momentum/all/assigned/pz_bias": 0.0032494592014700174, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0201075077056885, "metrics/particle/momentum/all/assigned/samples": 34768.0, "metrics/particle/momentum/all/efficiency": 0.8155752420425415, "metrics/particle/momentum/all/event/energy_bias": -4.51653528213501, "metrics/particle/momentum/all/event/p3_error_mean": 17.23255157470703, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.909353256225586, "metrics/particle/momentum/all/event/px_bias": -0.6210432052612305, "metrics/particle/momentum/all/event/py_bias": -0.627933144569397, "metrics/particle/momentum/all/event/pz_abs_mean": 12.526786804199219, "metrics/particle/momentum/all/event/pz_bias": -0.8658013343811035, "metrics/particle/momentum/all/event/pz_residual_rms": 20.963085174560547, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009476383216679096, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05449596419930458, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04375313222408295, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008554846979677677, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09611006826162338, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015141000039875507, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15398162603378296, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.005263648461550474, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8606826663017273, "metrics/particle/momentum/class_1/accepted/samples": 15851.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10045892745256424, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3082679808139801, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.363066703081131, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2986041307449341, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05569501966238022, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.185939311981201, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006713760085403919, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3599585294723511, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.016361718997359276, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.319437265396118, "metrics/particle/momentum/class_1/assigned/samples": 17129.0, "metrics/particle/momentum/class_1/efficiency": 0.9246339797973633, "metrics/particle/momentum/class_1/event/energy_bias": 1.354325532913208, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.012969017028809, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.4690327644348145, "metrics/particle/momentum/class_1/event/px_bias": 0.5094940662384033, "metrics/particle/momentum/class_1/event/py_bias": 0.160457581281662, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.801456451416016, "metrics/particle/momentum/class_1/event/pz_bias": -1.1375563144683838, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.648412704467773, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03933906927704811, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20782870054244995, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2103881537914276, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06180671975016594, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2487858235836029, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.038142621517181396, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9038652181625366, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.151437446475029, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.360759735107422, "metrics/particle/momentum/class_2/accepted/samples": 1206.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9389649629592896, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7559425234794617, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 78530.34375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 78529.828125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3881242573261261, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3960084.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07589460909366608, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0768797397613525, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05225098505616188, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.6360673904418945, "metrics/particle/momentum/class_2/assigned/samples": 2543.0, "metrics/particle/momentum/class_2/efficiency": 0.4173010289669037, "metrics/particle/momentum/class_2/event/energy_bias": -11.190774917602539, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.236379623413086, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.405407905578613, "metrics/particle/momentum/class_2/event/px_bias": -0.47016289830207825, "metrics/particle/momentum/class_2/event/py_bias": 0.2506488859653473, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.664688110351562, "metrics/particle/momentum/class_2/event/pz_bias": 0.8424574732780457, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.068452835083008, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029645061120390892, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14159971475601196, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14037735760211945, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.018690980970859528, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1813921481370926, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0003439085267018527, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.191145658493042, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04125158116221428, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5971474051475525, "metrics/particle/momentum/class_3/accepted/samples": 11631.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3149900734424591, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4385732412338257, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4587438106536865, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3127292394638062, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1581331491470337, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.822193145751953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008510638028383255, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5223329067230225, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.014988688752055168, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.758152723312378, "metrics/particle/momentum/class_3/assigned/samples": 14570.0, "metrics/particle/momentum/class_3/efficiency": 0.7657515406608582, "metrics/particle/momentum/class_3/event/energy_bias": 6.533233642578125, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.920835494995117, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.833507537841797, "metrics/particle/momentum/class_3/event/px_bias": -0.4430364668369293, "metrics/particle/momentum/class_3/event/py_bias": -1.0216792821884155, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.795825004577637, "metrics/particle/momentum/class_3/event/pz_bias": -0.707820475101471, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.535199165344238, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013257275335490704, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10775835812091827, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10705716162919998, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04476170241832733, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15619443356990814, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006666666828095913, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5240580439567566, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11515266448259354, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.662699580192566, "metrics/particle/momentum/class_4/accepted/samples": 300.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.21654382348060608, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7286067008972168, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5715662240982056, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4864197969436646, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1246458888053894, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.677075386047363, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014164306223392487, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7289320826530457, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12187457084655762, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.368637800216675, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8474576473236084, "metrics/particle/momentum/class_4/event/energy_bias": -0.9351897239685059, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.546854257583618, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7724764347076416, "metrics/particle/momentum/class_4/event/px_bias": -0.29822951555252075, "metrics/particle/momentum/class_4/event/py_bias": 0.02140931785106659, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5000030994415283, "metrics/particle/momentum/class_4/event/pz_bias": 0.04328326880931854, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.550307750701904, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009502267464995384, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03405187278985977, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03141648694872856, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005375663749873638, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08555357903242111, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1878388226032257, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.034896619617938995, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.46398767828941345, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.012595116160809994, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06276512891054153, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06008332222700119, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.016158629208803177, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.21910467743873596, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2822980284690857, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.006035221740603447, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9699777364730835, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": -0.27812957763671875, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9789744019508362, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7824019193649292, "metrics/particle/momentum/class_5/event/px_bias": 0.08089166134595871, "metrics/particle/momentum/class_5/event/py_bias": -0.038769666105508804, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.47094470262527466, "metrics/particle/momentum/class_5/event/pz_bias": 0.09383461624383926, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.8209503889083862, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3027443885803223, "step_valid_time": 3612.1150085926056, "step_total_time": 3613.4177532196045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974624431631188, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09702610999238404, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2897, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2880, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9507712504102396, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526471555409405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9517082785808147, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04735284445905952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9451919921234001, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9470568891811904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9461235216819974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9507712504102396, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9950422943720996, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10495651811942164, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2269, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2157, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.946883230904302, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9506390480387836, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9487574224763581, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0493609519612164, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.940737489025461, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9444689290436316, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9425995161645041, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.946883230904302, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9979911316732933, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10893444366005034, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1224, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.928679817905918, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9336384439359268, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9311525294788893, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06636155606407322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9233687405159332, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9282990083905416, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9258273107645493, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.928679817905918} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_2000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..0891afb5c059100236f4bb9132a8855dc2401214 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.094732853353543, "train": {"Classification_binary": 4.3430891036987305, "Classification": 0.03777946159243584, "Regression_pt": 0.3004496991634369, "Regression_eta": 0.01652592606842518, "Regression_sin_phi": 0.009457766078412533, "Regression_cos_phi": 0.011294160038232803, "Regression_energy": 0.30908137559890747, "Total": 15.079377174377441, "Total_unweighted": 5.027677492238581}, "train_group": {}, "valid": {"Classification_binary": 4.305825551350911, "Classification": 0.03506554663181305, "Regression_pt": 0.2783471743265788, "Regression_eta": 0.016933128237724304, "Regression_sin_phi": 0.010089311748743057, "Regression_cos_phi": 0.010809939354658127, "Regression_energy": 0.2824651797612508, "Total": 14.432852427164713, "Total_unweighted": 4.939535831411679, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.03483833000063896, "diagnostic/loss/cld_pf/Classification_binary": 4.283620834350586, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.010532011277973652, "diagnostic/loss/cld_pf/Regression_energy": 0.28514471650123596, "diagnostic/loss/cld_pf/Regression_eta": 0.015777217224240303, "diagnostic/loss/cld_pf/Regression_pt": 0.2816952168941498, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.009272759780287743, "diagnostic/regression/cld_pf/energy_pred_mean": 0.1331642121076584, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2794936001300812, "diagnostic/regression/cld_pf/energy_residual_mean": 0.10604492574930191, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5761110186576843, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.10510814934968948, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2832539975643158, "diagnostic/regression/cld_pf/pt_residual_mean": 0.1322752833366394, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5784449577331543, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.885430634021759, "metrics/particle/class_1/pid_accuracy": 0.9805194735527039, "metrics/particle/class_2/efficiency": 0.1956249326467514, "metrics/particle/class_2/pid_accuracy": 0.1991846263408661, "metrics/particle/class_3/efficiency": 0.6681390404701233, "metrics/particle/class_3/pid_accuracy": 0.9802979826927185, "metrics/particle/class_4/efficiency": 0.8779011368751526, "metrics/particle/class_4/pid_accuracy": 0.01149425283074379, "metrics/particle/class_5/efficiency": 0.9514563083648682, "metrics/particle/class_5/pid_accuracy": 0.1081632673740387, "metrics/particle/count/bias_mean": 1.9052287340164185, "metrics/particle/count/mae": 5.191503047943115, "metrics/particle/count/prediction_mean": 72.47058868408203, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.1164383739233017, "metrics/particle/event/energy_response_mean": 1.031288743019104, "metrics/particle/event/met_abs_error": 17.185386657714844, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1111021637916565, "metrics/particle/event/scalar_pt_response_mean": 1.0485645532608032, "metrics/particle/event/vector_pt_closure": 0.12410995364189148, "metrics/particle/matched/delta_eta_abs_mean": 0.01709129847586155, "metrics/particle/matched/delta_phi_abs_mean": 0.013449015095829964, "metrics/particle/matched/delta_r_mean": 0.024339936673641205, "metrics/particle/matched/energy_relative_abs_mean": 0.1005886122584343, "metrics/particle/matched/pid_accuracy": 0.9476531147956848, "metrics/particle/matched/pt_relative_abs_mean": 0.10262484103441238, "metrics/particle/matching/duplicate_fraction": 0.05106421187520027, "metrics/particle/matching/efficiency": 0.7380169630050659, "metrics/particle/matching/f1": 0.7281866073608398, "metrics/particle/matching/purity": 0.7186146974563599, "metrics/particle/matching/target_coverage_dr0p05": 0.6829528212547302, "metrics/particle/matching/target_coverage_dr0p10": 0.78718101978302, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6253786087036133, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7890797853469849, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02986815758049488, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09745270013809204, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.10302610695362091, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03394555673003197, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15508975088596344, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005628253798931837, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4659517705440521, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07533448934555054, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7893718481063843, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7107.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4867077171802521, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.35292911529541, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.923319101333618, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.78049635887146, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2169419378042221, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 28.15016746520996, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.030904853716492653, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3719215393066406, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.025567682459950447, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.849974155426025, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9869.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7010949850082397, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.4741969108581543, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 16.504573822021484, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.5657453536987305, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.2699706554412842, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.14993372559547424, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.040847778320312, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.677103579044342, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 24.325103759765625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0368807390332222, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10847640037536621, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.11430409550666809, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02762124501168728, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1676034778356552, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024330900982022285, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0939666032791138, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3802870213985443, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.454212188720703, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1233.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.1583693027496338, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.763014793395996, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10159.2275390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10159.025390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3829039931297302, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 513859.59375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.12451209872961044, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.468694686889648, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3308199942111969, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.203609466552734, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2562.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4485267400741577, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.48712956905365, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.345413208007812, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.081972599029541, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.2312953770160675, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08255808055400848, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.06981086730957, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.0336328744888306, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.877683639526367, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021166285499930382, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10323493927717209, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.10422355681657791, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.04239148646593094, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15807434916496277, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006068630609661341, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13735495507717133, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.025391273200511932, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5623976588249207, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18126.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.263398677110672, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4053401947021484, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2134149074554443, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0723217725753784, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16046646237373352, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.877105712890625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008468939922749996, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.535882294178009, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.016152748838067055, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.1169960498809814, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22553.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7928093671798706, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.778602123260498, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.006052017211914, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.775846481323242, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6416501402854919, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.4306514263153076, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 9.157108306884766, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.06073111295700073, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 16.36347198486328, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024235133081674576, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.10192641615867615, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10437163710594177, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.039435356855392456, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15773741900920868, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030605304054915905, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.27016082406044006, "metrics/particle/momentum/all/accepted/pz_bias": -0.055336590856313705, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.277612566947937, "metrics/particle/momentum/all/accepted/samples": 26466.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3919360637664795, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7720826864242554, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 745.6025390625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 745.4564819335938, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.19268809258937836, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 139059.0625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.023296363651752472, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.05974280834198, "metrics/particle/momentum/all/assigned/pz_bias": -0.04185288026928902, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.879089832305908, "metrics/particle/momentum/all/assigned/samples": 34984.0, "metrics/particle/momentum/all/efficiency": 0.7403283715248108, "metrics/particle/momentum/all/event/energy_bias": 10.739928245544434, "metrics/particle/momentum/all/event/p3_error_mean": 36.1766357421875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 20.444053649902344, "metrics/particle/momentum/all/event/px_bias": 2.142916202545166, "metrics/particle/momentum/all/event/py_bias": -1.3632758855819702, "metrics/particle/momentum/all/event/pz_abs_mean": 25.298507690429688, "metrics/particle/momentum/all/event/pz_bias": 0.41726037859916687, "metrics/particle/momentum/all/event/pz_residual_rms": 37.88274002075195, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.016910428181290627, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05871398001909256, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.061912521719932556, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03570515662431717, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1098545491695404, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017800632631406188, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.19690819084644318, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03747830167412758, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.01021146774292, "metrics/particle/momentum/class_1/accepted/samples": 15168.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.15554684400558472, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6632635593414307, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.918854296207428, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.8606808185577393, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09154559671878815, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.735413551330566, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010969134978950024, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5774013996124268, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.03434952720999718, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.321430206298828, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.8847926259040833, "metrics/particle/momentum/class_1/event/energy_bias": 23.092288970947266, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.064456939697266, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 14.934610366821289, "metrics/particle/momentum/class_1/event/px_bias": 0.33431464433670044, "metrics/particle/momentum/class_1/event/py_bias": 0.17195668816566467, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.460270881652832, "metrics/particle/momentum/class_1/event/pz_bias": -0.6100237369537354, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.89496612548828, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04614122211933136, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22980089485645294, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22531026601791382, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0772230252623558, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2670389413833618, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.06164383515715599, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.2943928241729736, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.7150324583053589, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.365208625793457, "metrics/particle/momentum/class_2/accepted/samples": 584.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.524430513381958, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.4065215587615967, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10075.6103515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10074.8681640625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.6034082174301147, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 511865.5625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.14136327803134918, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 5.13466739654541, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3877372145652771, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.552000999450684, "metrics/particle/momentum/class_2/assigned/samples": 2582.0, "metrics/particle/momentum/class_2/efficiency": 0.20207612216472626, "metrics/particle/momentum/class_2/event/energy_bias": -21.172685623168945, "metrics/particle/momentum/class_2/event/p3_error_mean": 25.339750289916992, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.002596855163574, "metrics/particle/momentum/class_2/event/px_bias": -0.011671394109725952, "metrics/particle/momentum/class_2/event/py_bias": -0.3814307451248169, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.526325225830078, "metrics/particle/momentum/class_2/event/pz_bias": 1.3577295541763306, "metrics/particle/momentum/class_2/event/pz_residual_rms": 30.67254066467285, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03414201736450195, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15986604988574982, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1616630107164383, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.052575960755348206, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20396430790424347, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014651298988610506, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2518831491470337, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04827450215816498, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9081828594207764, "metrics/particle/momentum/class_3/accepted/samples": 10238.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4762732982635498, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.0830957889556885, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.507901668548584, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.361356258392334, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24212811887264252, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 33.2115364074707, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01737242192029953, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.917845606803894, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01130636315792799, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.88210916519165, "metrics/particle/momentum/class_3/assigned/samples": 14736.0, "metrics/particle/momentum/class_3/efficiency": 0.6740404367446899, "metrics/particle/momentum/class_3/event/energy_bias": 17.317869186401367, "metrics/particle/momentum/class_3/event/p3_error_mean": 18.662029266357422, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.645830154418945, "metrics/particle/momentum/class_3/event/px_bias": 1.4684494733810425, "metrics/particle/momentum/class_3/event/py_bias": -1.1224305629730225, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.959930419921875, "metrics/particle/momentum/class_3/event/pz_bias": -0.7347168922424316, "metrics/particle/momentum/class_3/event/pz_residual_rms": 24.420827865600586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020071808248758316, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10141259431838989, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10244052857160568, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.02192498743534088, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15535344183444977, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009677419438958168, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6518142223358154, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.09395904839038849, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.7893779277801514, "metrics/particle/momentum/class_4/accepted/samples": 310.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.23172827064990997, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.388161659240723, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.186840534210205, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.1066155433654785, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11299435049295425, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.99045181274414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9587119817733765, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05008390545845032, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.706986665725708, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8757061958312988, "metrics/particle/momentum/class_4/event/energy_bias": -6.0824456214904785, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.618635177612305, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.31225061416626, "metrics/particle/momentum/class_4/event/px_bias": 0.3024272620677948, "metrics/particle/momentum/class_4/event/py_bias": 0.5345402956008911, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.9105892181396484, "metrics/particle/momentum/class_4/event/pz_bias": 0.4605177044868469, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.10774040222168, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013222471810877323, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028074242174625397, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028716541826725006, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012946248054504395, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04988228902220726, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2566557824611664, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08061101287603378, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.570173978805542, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05259789526462555, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.053845059126615524, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.054366711527109146, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.012539840303361416, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15084312856197357, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.32074612379074097, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.13415062427520752, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9227848052978516, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -2.415097713470459, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.28678560256958, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.436457395553589, "metrics/particle/momentum/class_5/event/px_bias": 0.04939614236354828, "metrics/particle/momentum/class_5/event/py_bias": -0.5659116506576538, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.9183917045593262, "metrics/particle/momentum/class_5/event/pz_bias": -0.05624626204371452, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.777978420257568, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.094877004623413, "step_valid_time": 467.4099032878876, "step_total_time": 468.50478076934814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0114596346900586, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23790398353830655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2729, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.895635050869708, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8760834670947031, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8857513794222655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12391653290529694, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8293403347554972, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8112359550561797, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8201882505679974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.895635050869708, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0232214018791872, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25525966643744147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2324, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2006, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1885, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8805970149253731, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8631669535283993, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8717948717948718, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1368330464716007, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8274802458296752, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8111015490533563, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8192090395480226, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8805970149253731, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0213515536840212, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24673623079642426, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1350, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1100, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.834597875569044, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8148148148148148, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8245877061469266, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.18518518518518517, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.779969650986343, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7614814814814815, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7706146926536732, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.834597875569044} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_20000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..7837991a0b569035c815289860202211b84ae23b --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9473285335354301, "train": {"Classification_binary": 3.07041597366333, "Classification": 0.021613432094454765, "Regression_pt": 0.18245533108711243, "Regression_eta": 0.020185908302664757, "Regression_sin_phi": 0.00869523361325264, "Regression_cos_phi": 0.008309553377330303, "Regression_energy": 0.1958470493555069, "Total": 10.455564498901367, "Total_unweighted": 3.507522481493652}, "train_group": {}, "valid": {"Classification_binary": 2.9358749389648438, "Classification": 0.02071507026751836, "Regression_pt": 0.16094102462132773, "Regression_eta": 0.012491424878438314, "Regression_sin_phi": 0.007365352163712184, "Regression_cos_phi": 0.00769599402944247, "Regression_energy": 0.16731045643488565, "Total": 9.210744222005209, "Total_unweighted": 3.3123942613601685, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020942892879247665, "diagnostic/loss/cld_pf/Classification_binary": 2.936339855194092, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0073774135671556, "diagnostic/loss/cld_pf/Regression_energy": 0.16682161390781403, "diagnostic/loss/cld_pf/Regression_eta": 0.011837485246360302, "diagnostic/loss/cld_pf/Regression_pt": 0.1615249216556549, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0067890300415456295, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09637507051229477, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21577264368534088, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06925579905509949, "diagnostic/regression/cld_pf/energy_residual_rms": 0.46229663491249084, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.054434336721897125, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20822320878505707, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08160148561000824, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4542945623397827, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9278623461723328, "metrics/particle/class_1/pid_accuracy": 0.9827942252159119, "metrics/particle/class_2/efficiency": 0.4117580056190491, "metrics/particle/class_2/pid_accuracy": 0.6934145092964172, "metrics/particle/class_3/efficiency": 0.7573839426040649, "metrics/particle/class_3/pid_accuracy": 0.9824164509773254, "metrics/particle/class_4/efficiency": 0.8577194809913635, "metrics/particle/class_4/pid_accuracy": 0.28823530673980713, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.5496957302093506, "metrics/particle/count/bias_mean": 0.06470588594675064, "metrics/particle/count/mae": 3.4882352352142334, "metrics/particle/count/prediction_mean": 70.63006591796875, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06147639825940132, "metrics/particle/event/energy_response_mean": 1.0127407312393188, "metrics/particle/event/met_abs_error": 6.405582904815674, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051338620483875275, "metrics/particle/event/scalar_pt_response_mean": 1.0178216695785522, "metrics/particle/event/vector_pt_closure": 0.0553501658141613, "metrics/particle/matched/delta_eta_abs_mean": 0.01235098484903574, "metrics/particle/matched/delta_phi_abs_mean": 0.0107917794957757, "metrics/particle/matched/delta_r_mean": 0.018405282869935036, "metrics/particle/matched/energy_relative_abs_mean": 0.0968707725405693, "metrics/particle/matched/pid_accuracy": 0.9615870714187622, "metrics/particle/matched/pt_relative_abs_mean": 0.0890701562166214, "metrics/particle/matching/duplicate_fraction": 0.03849570453166962, "metrics/particle/matching/efficiency": 0.8135507106781006, "metrics/particle/matching/f1": 0.8131778836250305, "metrics/particle/matching/purity": 0.8128053545951843, "metrics/particle/matching/target_coverage_dr0p05": 0.7841059565544128, "metrics/particle/matching/target_coverage_dr0p10": 0.8517853021621704, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7008382081985474, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8526096343994141, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020792901515960693, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09348266571760178, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08859705179929733, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.023820532485842705, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14246124029159546, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005604682955890894, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.42231127619743347, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04036474972963333, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6478604078292847, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8029.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3018029034137726, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4489203691482544, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5043981075286865, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3909170627593994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13625748455524445, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.981380462646484, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018174434080719948, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8917005062103271, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.011765514500439167, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4024720191955566, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9849.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7920489311218262, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.5856568813323975, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.62508487701416, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.7397735118865967, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2511644959449768, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.18606987595558167, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.545396327972412, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.973601222038269, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.680351257324219, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029000021517276764, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10394799709320068, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10260040313005447, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019662801176309586, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1565966159105301, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02522638998925686, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.023861050605774, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16478385031223297, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0737082958221436, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1546.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8425644636154175, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7952908277511597, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 25258.7109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 25258.546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2937743067741394, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1280302.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09027237445116043, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1548008918762207, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3023292124271393, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.819244384765625, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2570.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5623863339424133, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.7306448221206665, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.29838228225708, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2872257232666016, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.15551021695137024, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07158678025007248, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.1256914138793945, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6001914739608765, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.837814331054688, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016530169174075127, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09880012273788452, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08986543864011765, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017869682982563972, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14574803411960602, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005119541310705245, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12553316354751587, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.022100673988461494, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5155125260353088, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19533.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1674354523420334, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7140737175941467, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5489056706428528, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4310440719127655, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10435168445110321, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.244621276855469, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0039964476600289345, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.30246907472610474, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.012975054793059826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3593683242797852, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22520.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8543498516082764, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8021349906921387, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.411212921142578, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.077749252319336, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8401784300804138, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.4125667810440063, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.787864923477173, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.11578293144702911, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.902994632720947, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01836828514933586, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09760680049657822, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09019196033477783, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019606372341513634, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1454489827156067, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003229352645576, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25510740280151367, "metrics/particle/momentum/all/accepted/pz_bias": -0.034716807305812836, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1954659223556519, "metrics/particle/momentum/all/accepted/samples": 29108.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25497278571128845, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0007513761520386, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1858.7271728515625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1858.60693359375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12727896869182587, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 347235.4375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01433927658945322, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6783766150474548, "metrics/particle/momentum/all/assigned/pz_bias": -0.027284864336252213, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.189927577972412, "metrics/particle/momentum/all/assigned/samples": 34939.0, "metrics/particle/momentum/all/efficiency": 0.814232587814331, "metrics/particle/momentum/all/event/energy_bias": 5.118436813354492, "metrics/particle/momentum/all/event/p3_error_mean": 17.718734741210938, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.197834014892578, "metrics/particle/momentum/all/event/px_bias": 1.2468531131744385, "metrics/particle/momentum/all/event/py_bias": 1.2980836629867554, "metrics/particle/momentum/all/event/pz_abs_mean": 13.085224151611328, "metrics/particle/momentum/all/event/pz_bias": -1.4580097198486328, "metrics/particle/momentum/all/event/pz_residual_rms": 21.20936393737793, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010358228348195553, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05629875138401985, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04395793750882149, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007149241399019957, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09552560746669769, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001449549337849021, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15979677438735962, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.009534967131912708, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8669294118881226, "metrics/particle/momentum/class_1/accepted/samples": 15867.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10000360757112503, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30114755034446716, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.37858596444129944, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.31111493706703186, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.055416204035282135, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.602025508880615, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006941608618944883, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3748375475406647, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01574035733938217, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.380546808242798, "metrics/particle/momentum/class_1/assigned/samples": 17143.0, "metrics/particle/momentum/class_1/efficiency": 0.9255672693252563, "metrics/particle/momentum/class_1/event/energy_bias": 1.1817995309829712, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.845680236816406, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.378770351409912, "metrics/particle/momentum/class_1/event/px_bias": 0.4156132936477661, "metrics/particle/momentum/class_1/event/py_bias": 0.17474500834941864, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.828287124633789, "metrics/particle/momentum/class_1/event/pz_bias": 0.21434111893177032, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.65045928955078, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03972375765442848, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2027818113565445, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20574340224266052, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04086831212043762, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24556899070739746, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0458715595304966, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9321032762527466, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.33137670159339905, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.309351444244385, "metrics/particle/momentum/class_2/accepted/samples": 1199.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9244365096092224, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9449157118797302, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 24698.828125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 24698.36328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3957382142543793, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1266095.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0833333358168602, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.345860242843628, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.039975207298994064, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.97690486907959, "metrics/particle/momentum/class_2/assigned/samples": 2628.0, "metrics/particle/momentum/class_2/efficiency": 0.4148789048194885, "metrics/particle/momentum/class_2/event/energy_bias": -4.284488677978516, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.951807022094727, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.25753116607666, "metrics/particle/momentum/class_2/event/px_bias": 0.556283712387085, "metrics/particle/momentum/class_2/event/py_bias": 0.7426345348358154, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.640166282653809, "metrics/particle/momentum/class_2/event/pz_bias": -1.055827260017395, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.698328018188477, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027383413165807724, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14400246739387512, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14203675091266632, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04238249361515045, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1836523711681366, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012097122380509973, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.205149307847023, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.041172295808792114, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7402459979057312, "metrics/particle/momentum/class_3/accepted/samples": 11573.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3196275532245636, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.839945912361145, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8226654529571533, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6997276544570923, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1643330305814743, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.359962463378906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010928215458989143, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.561173141002655, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.039787743240594864, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.476550817489624, "metrics/particle/momentum/class_3/assigned/samples": 14641.0, "metrics/particle/momentum/class_3/efficiency": 0.7619329690933228, "metrics/particle/momentum/class_3/event/energy_bias": 9.455602645874023, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.5711669921875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.991995334625244, "metrics/particle/momentum/class_3/event/px_bias": 0.16644270718097687, "metrics/particle/momentum/class_3/event/py_bias": 0.12789449095726013, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.461475849151611, "metrics/particle/momentum/class_3/event/pz_bias": -0.6068096160888672, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.049860954284668, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014620642177760601, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11094477027654648, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10930968075990677, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05032505467534065, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15844300389289856, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0066006602719426155, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5587087869644165, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06703140586614609, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.841020941734314, "metrics/particle/momentum/class_4/accepted/samples": 303.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2287486046552658, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0459345579147339, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0287489891052246, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.9703128337860107, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12429378181695938, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.547024250030518, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6228015422821045, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.062042299658060074, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8360439538955688, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8559321761131287, "metrics/particle/momentum/class_4/event/energy_bias": -0.9389407634735107, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.811941385269165, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9405717849731445, "metrics/particle/momentum/class_4/event/px_bias": -0.0034947171807289124, "metrics/particle/momentum/class_4/event/py_bias": 0.2233058214187622, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7082527875900269, "metrics/particle/momentum/class_4/event/pz_bias": -0.07232349365949631, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.1139349937438965, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008090218529105186, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.027435382828116417, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02547190524637699, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0031643721740692854, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06745553016662598, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18130585551261902, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.03462907671928406, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4887836277484894, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.023519445210695267, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06088012084364891, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05873984470963478, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008522157557308674, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.040462426841259, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.18914088606834412, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.26838988065719604, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.10314800590276718, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9730423092842102, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.29553529620170593, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9526041746139526, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7658535838127136, "metrics/particle/momentum/class_5/event/px_bias": 0.11200816929340363, "metrics/particle/momentum/class_5/event/py_bias": 0.029503751546144485, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.46240895986557007, "metrics/particle/momentum/class_5/event/pz_bias": 0.06260943412780762, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7729194164276123, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.308302402496338, "step_valid_time": 3772.029218673706, "step_total_time": 3773.337521314621, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064232102693875, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09455162743932455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3057, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9596324253363965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9564932940791626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9580602883355177, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04350670592083742, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9553659337052839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952240758913968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9538007863695938, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9596324253363965, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0084519561654588, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10542900011746104, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2284, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2162, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9490781387181738, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.946584938704028, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.947829899167032, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053415061295971976, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9429323968393327, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9404553415061296, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9416922402455064, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9490781387181738, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0140883849356743, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10978137628745899, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1228, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9317147192716236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9338403041825095, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9327763007975693, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0661596958174905, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9233687405159332, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9254752851711027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.924420812761109, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9317147192716236} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_21000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..da68b563c32e8b2b67c676ad3bd652cdc2da411b --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9946949602122016, "train": {"Classification_binary": 2.9508867263793945, "Classification": 0.019181620329618454, "Regression_pt": 0.17057611048221588, "Regression_eta": 0.021422958001494408, "Regression_sin_phi": 0.009883943945169449, "Regression_cos_phi": 0.008032410405576229, "Regression_energy": 0.1831645667552948, "Total": 10.114380836486816, "Total_unweighted": 3.3631483362987638}, "train_group": {}, "valid": {"Classification_binary": 2.9355646769205728, "Classification": 0.020554269353548687, "Regression_pt": 0.16285633047421774, "Regression_eta": 0.012552369385957718, "Regression_sin_phi": 0.007280000175038974, "Regression_cos_phi": 0.007734579344590505, "Regression_energy": 0.16912192106246948, "Total": 9.244085947672525, "Total_unweighted": 3.315664146716396, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020578628405928612, "diagnostic/loss/cld_pf/Classification_binary": 2.928873062133789, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0073517123237252235, "diagnostic/loss/cld_pf/Regression_energy": 0.1700534224510193, "diagnostic/loss/cld_pf/Regression_eta": 0.011725466698408127, "diagnostic/loss/cld_pf/Regression_pt": 0.16482622921466827, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006683606654405594, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08652166277170181, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2143150418996811, "diagnostic/regression/cld_pf/energy_residual_mean": 0.059402380138635635, "diagnostic/regression/cld_pf/energy_residual_rms": 0.45742151141166687, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04976470023393631, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20910021662712097, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07693184167146683, "diagnostic/regression/cld_pf/pt_residual_rms": 0.45094186067581177, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9271340370178223, "metrics/particle/class_1/pid_accuracy": 0.9823879599571228, "metrics/particle/class_2/efficiency": 0.4133530855178833, "metrics/particle/class_2/pid_accuracy": 0.6979051828384399, "metrics/particle/class_3/efficiency": 0.7469013929367065, "metrics/particle/class_3/pid_accuracy": 0.9811986684799194, "metrics/particle/class_4/efficiency": 0.8446013927459717, "metrics/particle/class_4/pid_accuracy": 0.2735961675643921, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.57230144739151, "metrics/particle/count/bias_mean": -0.886274516582489, "metrics/particle/count/mae": 3.7058823108673096, "metrics/particle/count/prediction_mean": 69.67908477783203, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06197978928685188, "metrics/particle/event/energy_response_mean": 0.9913408160209656, "metrics/particle/event/met_abs_error": 6.537990093231201, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05103836953639984, "metrics/particle/event/scalar_pt_response_mean": 1.001585602760315, "metrics/particle/event/vector_pt_closure": 0.055911917239427567, "metrics/particle/matched/delta_eta_abs_mean": 0.012703891843557358, "metrics/particle/matched/delta_phi_abs_mean": 0.01187159027904272, "metrics/particle/matched/delta_r_mean": 0.019419599324464798, "metrics/particle/matched/energy_relative_abs_mean": 0.09549345821142197, "metrics/particle/matched/pid_accuracy": 0.9610050320625305, "metrics/particle/matched/pt_relative_abs_mean": 0.08959327638149261, "metrics/particle/matching/duplicate_fraction": 0.03873031213879585, "metrics/particle/matching/efficiency": 0.8087713718414307, "metrics/particle/matching/f1": 0.8138824105262756, "metrics/particle/matching/purity": 0.8190584182739258, "metrics/particle/matching/target_coverage_dr0p05": 0.7635437250137329, "metrics/particle/matching/target_coverage_dr0p10": 0.8458945155143738, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6969202756881714, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8479970097541809, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023117614910006523, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09116663038730621, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0884077176451683, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01963123306632042, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1417812705039978, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0057839807122945786, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.41877785325050354, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.008578210137784481, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6962404251098633, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7953.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2962283194065094, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2094732522964478, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2789791822433472, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1633578538894653, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13256601989269257, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.197778701782227, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01880587823688984, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8715137243270874, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02159027010202408, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.02579402923584, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9731.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7845516204833984, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.537585437297821, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.80964469909668, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.802842617034912, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.8173992037773132, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.37509527802467346, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.680534362792969, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.6395517587661743, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.447230339050293, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03161410614848137, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10038616508245468, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10185492038726807, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01873168721795082, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1570645570755005, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02069425955414772, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9697452783584595, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1937926560640335, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0780622959136963, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1498.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7623874545097351, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4805474281311035, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 25539.603515625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 25539.431640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2809983789920807, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1272731.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08655394613742828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.159975528717041, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.38416096568107605, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.706798553466797, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2484.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5449254512786865, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.7332682609558105, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.386223793029785, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1249136924743652, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03587467223405838, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.21058207750320435, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.217599391937256, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6271961331367493, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.461368560791016, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016986891627311707, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09847372770309448, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09104634821414948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018819661810994148, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14664921164512634, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009755596402101219, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1352415829896927, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00450832350179553, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6082548499107361, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19476.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17066709697246552, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8442022204399109, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6592043042182922, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5408731698989868, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10368385910987854, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.06332015991211, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0052243806421756744, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.32268521189689636, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.00393554475158453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5876384973526, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22395.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8518567085266113, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.2216481864452362, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.094108581542969, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.818819522857666, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.37716981768608093, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.1538901925086975, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6300947666168213, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.7699142098426819, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.1382527351379395, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01942991092801094, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09656380116939545, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0908806324005127, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019038233906030655, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14588961005210876, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033186988439410925, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2564104199409485, "metrics/particle/momentum/all/accepted/pz_bias": -0.004641849547624588, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2372485399246216, "metrics/particle/momentum/all/accepted/samples": 28927.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2484385222196579, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.992573618888855, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1833.7933349609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1833.671875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12453047931194305, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 340966.53125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014880092814564705, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6806303262710571, "metrics/particle/momentum/all/assigned/pz_bias": -0.0310954712331295, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3117318153381348, "metrics/particle/momentum/all/assigned/samples": 34610.0, "metrics/particle/momentum/all/efficiency": 0.8091694712638855, "metrics/particle/momentum/all/event/energy_bias": -1.4173309803009033, "metrics/particle/momentum/all/event/p3_error_mean": 16.968692779541016, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.955516815185547, "metrics/particle/momentum/all/event/px_bias": 1.1586942672729492, "metrics/particle/momentum/all/event/py_bias": 0.7395676374435425, "metrics/particle/momentum/all/event/pz_abs_mean": 12.299386978149414, "metrics/particle/momentum/all/event/pz_bias": -2.0366621017456055, "metrics/particle/momentum/all/event/pz_residual_rms": 20.460865020751953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010598843917250633, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05659953132271767, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04622313752770424, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008778582327067852, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09863211214542389, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019562062807381153, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17013005912303925, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0057516093365848064, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9622775912284851, "metrics/particle/momentum/class_1/accepted/samples": 15847.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10164706408977509, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3477415144443512, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4688973128795624, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4028192460536957, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05538048595190048, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.146993637084961, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007061157841235399, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.387909471988678, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.011016578413546085, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.770328998565674, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9244006276130676, "metrics/particle/momentum/class_1/event/energy_bias": -0.49734094738960266, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.230018615722656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.645449638366699, "metrics/particle/momentum/class_1/event/px_bias": 0.3471977710723877, "metrics/particle/momentum/class_1/event/py_bias": -0.04001709818840027, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.969366073608398, "metrics/particle/momentum/class_1/event/pz_bias": -0.29939004778862, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.015247344970703, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.039422232657670975, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20102627575397491, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20133604109287262, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04193352535367012, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24154682457447052, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04500000178813934, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.89409339427948, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.23715634644031525, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.421541690826416, "metrics/particle/momentum/class_2/accepted/samples": 1200.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8573111295700073, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8365449905395508, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 25222.87890625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 25222.412109375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.38926440477371216, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1264863.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08946321904659271, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.2932517528533936, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3267613351345062, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.784497261047363, "metrics/particle/momentum/class_2/assigned/samples": 2515.0, "metrics/particle/momentum/class_2/efficiency": 0.41522490978240967, "metrics/particle/momentum/class_2/event/energy_bias": -5.888966083526611, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.80053997039795, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.134529113769531, "metrics/particle/momentum/class_2/event/px_bias": -0.33865028619766235, "metrics/particle/momentum/class_2/event/py_bias": 0.7488028407096863, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.649497985839844, "metrics/particle/momentum/class_2/event/pz_bias": -2.093503952026367, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.748966217041016, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029857242479920387, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14128315448760986, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14124131202697754, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03866179287433624, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18279273808002472, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000700770819094032, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1953483372926712, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0013310573995113373, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6582978367805481, "metrics/particle/momentum/class_3/accepted/samples": 11416.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3201509714126587, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7884838581085205, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6341309547424316, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5052506923675537, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16144677996635437, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.028060913085938, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01143292710185051, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5772007703781128, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.006223432254046202, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.55344295501709, "metrics/particle/momentum/class_3/assigned/samples": 14432.0, "metrics/particle/momentum/class_3/efficiency": 0.7515965700149536, "metrics/particle/momentum/class_3/event/energy_bias": 6.3770341873168945, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.502830505371094, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.412140369415283, "metrics/particle/momentum/class_3/event/px_bias": 1.3235929012298584, "metrics/particle/momentum/class_3/event/py_bias": -0.18455420434474945, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.985589504241943, "metrics/particle/momentum/class_3/event/pz_bias": -0.11237987875938416, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.012686729431152, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014943541958928108, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.122227244079113, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12257415056228638, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06229110434651375, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17194963991641998, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010101010091602802, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6018114686012268, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.15383292734622955, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9050095081329346, "metrics/particle/momentum/class_4/accepted/samples": 297.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.20713594555854797, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.327653408050537, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.1671998500823975, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1029083728790283, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1355932205915451, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.526171684265137, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7073035836219788, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08374296128749847, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0101683139801025, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8389830589294434, "metrics/particle/momentum/class_4/event/energy_bias": -1.2376782894134521, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3842179775238037, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.772428035736084, "metrics/particle/momentum/class_4/event/px_bias": -0.20191654562950134, "metrics/particle/momentum/class_4/event/py_bias": 0.11908166110515594, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2918528318405151, "metrics/particle/momentum/class_4/event/pz_bias": 0.42953503131866455, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.558258533477783, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008945603854954243, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.035609882324934006, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03585239127278328, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01234441064298153, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08725816011428833, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2358444184064865, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01027931459248066, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8545452952384949, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.039023272693157196, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05072687938809395, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05334910377860069, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.005813754163682461, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13441626727581024, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2677658200263977, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.031548187136650085, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9009017944335938, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.1703799068927765, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0647209882736206, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.870966911315918, "metrics/particle/momentum/class_5/event/px_bias": 0.028470391407608986, "metrics/particle/momentum/class_5/event/py_bias": 0.0962543711066246, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.49258100986480713, "metrics/particle/momentum/class_5/event/pz_bias": 0.03907677158713341, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.8902583122253418, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.339916706085205, "step_valid_time": 3930.129557132721, "step_total_time": 3931.4694740772247, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0044919934143963, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09552049534205931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2887, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.95274040039383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568226763348715, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9547771748067752, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043177323665128546, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9474893337709223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9515491100856954, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9495148824206545, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.95274040039383, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0022378288051688, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11005999506183584, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2270, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2133, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9411764705882353, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9444933920704845, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9428320140721196, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055506607929515416, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9363476733977173, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.939647577092511, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9379947229551451, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9411764705882353, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005533852751205, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1131335944450248, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1316, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1213, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9203338391502276, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9217325227963525, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9210326499620349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07826747720364742, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9165402124430956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9179331306990881, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9172361427486713, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9203338391502276} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_22000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..cb7078e4d1844404e6eef6edbdca81cf56673704 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0420613868889732, "train": {"Classification_binary": 2.8855466842651367, "Classification": 0.020278170704841614, "Regression_pt": 0.1653871089220047, "Regression_eta": 0.013239117339253426, "Regression_sin_phi": 0.007505476474761963, "Regression_cos_phi": 0.009186532348394394, "Regression_energy": 0.17000779509544373, "Total": 9.369621276855469, "Total_unweighted": 3.2711508851498365}, "train_group": {}, "valid": {"Classification_binary": 2.880330721537272, "Classification": 0.02038208395242691, "Regression_pt": 0.15576313932736716, "Regression_eta": 0.0126905490954717, "Regression_sin_phi": 0.007238443319996198, "Regression_cos_phi": 0.007716767489910126, "Regression_energy": 0.16173880298932394, "Total": 9.018444061279297, "Total_unweighted": 3.245860507711768, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020343467593193054, "diagnostic/loss/cld_pf/Classification_binary": 2.8816747665405273, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007201837841421366, "diagnostic/loss/cld_pf/Regression_energy": 0.1627654731273651, "diagnostic/loss/cld_pf/Regression_eta": 0.011977004818618298, "diagnostic/loss/cld_pf/Regression_pt": 0.15773886442184448, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006795640103518963, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07458750158548355, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20888826251029968, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04746822640299797, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4468996524810791, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03368578106164932, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2030360996723175, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06085292249917984, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4394889175891876, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9292421936988831, "metrics/particle/class_1/pid_accuracy": 0.9844491481781006, "metrics/particle/class_2/efficiency": 0.41734078526496887, "metrics/particle/class_2/pid_accuracy": 0.703521728515625, "metrics/particle/class_3/efficiency": 0.7693389654159546, "metrics/particle/class_3/pid_accuracy": 0.9802045226097107, "metrics/particle/class_4/efficiency": 0.8405650854110718, "metrics/particle/class_4/pid_accuracy": 0.30132052302360535, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.5273833870887756, "metrics/particle/count/bias_mean": 0.21764706075191498, "metrics/particle/count/mae": 3.5117647647857666, "metrics/particle/count/prediction_mean": 70.78300476074219, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05735955014824867, "metrics/particle/event/energy_response_mean": 0.979225218296051, "metrics/particle/event/met_abs_error": 6.178864002227783, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048391807824373245, "metrics/particle/event/scalar_pt_response_mean": 0.9838695526123047, "metrics/particle/event/vector_pt_closure": 0.052774615585803986, "metrics/particle/matched/delta_eta_abs_mean": 0.011990882456302643, "metrics/particle/matched/delta_phi_abs_mean": 0.010841097682714462, "metrics/particle/matched/delta_r_mean": 0.018186187371611595, "metrics/particle/matched/energy_relative_abs_mean": 0.09452186524868011, "metrics/particle/matched/pid_accuracy": 0.9621621966362, "metrics/particle/matched/pt_relative_abs_mean": 0.08793384581804276, "metrics/particle/matching/duplicate_fraction": 0.03548542037606239, "metrics/particle/matching/efficiency": 0.8195526599884033, "metrics/particle/matching/f1": 0.8182907104492188, "metrics/particle/matching/purity": 0.8170326352119446, "metrics/particle/matching/target_coverage_dr0p05": 0.7850322127342224, "metrics/particle/matching/target_coverage_dr0p10": 0.8553512692451477, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.709711492061615, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8581114411354065, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022328488528728485, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0904962569475174, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08731690049171448, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.006617583800107241, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14029917120933533, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005801036953926086, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.40157461166381836, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.038365017622709274, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6557170152664185, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8102.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29935935139656067, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0010950565338135, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0016074180603027, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8710445165634155, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1307169646024704, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.499367713928223, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017239630222320557, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8255012631416321, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.013772040605545044, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.06378436088562, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9861.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7992502450942993, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8293370604515076, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.803702354431152, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.474240779876709, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11094506829977036, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.10282006859779358, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.651174068450928, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.2585107684135437, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.834086418151855, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03116176836192608, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09886537492275238, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09827475994825363, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.006453190464526415, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14855161309242249, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02363755740225315, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0430099964141846, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.31897127628326416, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4905099868774414, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1523.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8456335067749023, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7217185497283936, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 20886.78125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 20886.5859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28081920742988586, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1052277.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08428515493869781, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.178882598876953, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3757078945636749, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.465811729431152, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2539.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5540196299552917, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.3957887887954712, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.455904960632324, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.164196252822876, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.013905134052038193, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.019902125000953674, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.278304100036621, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.4806206226348877, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.61161231994629, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015226338990032673, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09723881632089615, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0891786590218544, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008919530548155308, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1443905234336853, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006605355301871896, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12042196094989777, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0023813482839614153, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5242094397544861, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19681.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16173264384269714, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6877085566520691, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.559440553188324, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4320474863052368, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10127704590559006, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.957277297973633, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0036360411904752254, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2849287986755371, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013659617863595486, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4807820320129395, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22552.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8608231544494629, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.2539703845977783, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.603346347808838, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.414498805999756, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.0335009098052979, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.33682936429977417, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.390471935272217, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0217374563217163, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.495702743530273, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0180179625749588, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09545927494764328, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08913666754961014, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007484226021915674, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14349202811717987, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032757797744125128, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2460959106683731, "metrics/particle/momentum/all/accepted/pz_bias": -0.025583814829587936, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2552324533462524, "metrics/particle/momentum/all/accepted/samples": 29306.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2502414584159851, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8512372374534607, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1517.9111328125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1517.77783203125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12262531369924545, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 283612.59375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013332569971680641, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6476643681526184, "metrics/particle/momentum/all/assigned/pz_bias": -0.022364290431141853, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2514567375183105, "metrics/particle/momentum/all/assigned/samples": 34952.0, "metrics/particle/momentum/all/efficiency": 0.8197711706161499, "metrics/particle/momentum/all/event/energy_bias": -5.479096412658691, "metrics/particle/momentum/all/event/p3_error_mean": 15.69189167022705, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.399137496948242, "metrics/particle/momentum/all/event/px_bias": 0.9086506366729736, "metrics/particle/momentum/all/event/py_bias": -0.21410715579986572, "metrics/particle/momentum/all/event/pz_abs_mean": 11.19355583190918, "metrics/particle/momentum/all/event/pz_bias": -1.7173938751220703, "metrics/particle/momentum/all/event/pz_residual_rms": 19.77713394165039, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010006657801568508, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05508166551589966, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04392191022634506, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006218734662979841, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09561342000961304, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018884552409872413, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15577350556850433, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.012894322164356709, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8851518630981445, "metrics/particle/momentum/class_1/accepted/samples": 15886.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0983617827296257, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30321627855300903, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3370792865753174, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.27320408821105957, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.053344227373600006, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.324012279510498, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006886891555041075, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.37208858132362366, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0033232455607503653, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.743143081665039, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.9266756176948547, "metrics/particle/momentum/class_1/event/energy_bias": 3.512347459793091, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.006269454956055, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.66918420791626, "metrics/particle/momentum/class_1/event/px_bias": 0.6121820211410522, "metrics/particle/momentum/class_1/event/py_bias": -0.05684638023376465, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.693718910217285, "metrics/particle/momentum/class_1/event/pz_bias": -0.40526074171066284, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.224449157714844, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038409650325775146, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20544873178005219, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2073056995868683, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06122985854744911, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2461017221212387, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04616652801632881, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9448230266571045, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.34521228075027466, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.649507999420166, "metrics/particle/momentum/class_2/accepted/samples": 1213.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9622124433517456, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8637800812721252, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20300.822265625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20300.30078125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3931852877140045, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1037468.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08537518978118896, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.3538386821746826, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19696353375911713, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.926344871520996, "metrics/particle/momentum/class_2/assigned/samples": 2612.0, "metrics/particle/momentum/class_2/efficiency": 0.41972318291664124, "metrics/particle/momentum/class_2/event/energy_bias": -11.266895294189453, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.025405883789062, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.6965484619140625, "metrics/particle/momentum/class_2/event/px_bias": 0.06024351716041565, "metrics/particle/momentum/class_2/event/py_bias": 0.03847602754831314, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.967328071594238, "metrics/particle/momentum/class_2/event/pz_bias": -1.8654569387435913, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.959574699401855, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02699575014412403, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13915576040744781, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1382914036512375, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01540256105363369, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17876723408699036, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006810249178670347, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1870672106742859, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.015421492978930473, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7310457825660706, "metrics/particle/momentum/class_3/accepted/samples": 11747.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3038633465766907, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.486087441444397, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5011577606201172, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3538222312927246, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1557326763868332, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.412284851074219, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00817494373768568, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4912722706794739, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.018485167995095253, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2484188079833984, "metrics/particle/momentum/class_3/assigned/samples": 14679.0, "metrics/particle/momentum/class_3/efficiency": 0.7733886241912842, "metrics/particle/momentum/class_3/event/energy_bias": 4.1243896484375, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.599483489990234, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.559647083282471, "metrics/particle/momentum/class_3/event/px_bias": 0.5005413293838501, "metrics/particle/momentum/class_3/event/py_bias": -0.41210484504699707, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.594260215759277, "metrics/particle/momentum/class_3/event/pz_bias": 0.045416779816150665, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.033319473266602, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013065069913864136, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11128828674554825, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1112784594297409, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.043197229504585266, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.164893239736557, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006825938355177641, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5272032618522644, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1764794886112213, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0152838230133057, "metrics/particle/momentum/class_4/accepted/samples": 293.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.23568260669708252, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3468812704086304, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3085987567901611, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2270596027374268, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.108574390411377, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7042412161827087, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1978592723608017, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.442121982574463, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8276836276054382, "metrics/particle/momentum/class_4/event/energy_bias": -1.4612771272659302, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.356666088104248, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7411997318267822, "metrics/particle/momentum/class_4/event/px_bias": -0.31568825244903564, "metrics/particle/momentum/class_4/event/py_bias": 0.17913439869880676, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2871023416519165, "metrics/particle/momentum/class_4/event/pz_bias": 0.4145144820213318, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.28057861328125, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009165016934275627, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03606482222676277, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03545041009783745, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007323681842535734, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08958017826080322, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15837788581848145, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.01958213932812214, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4754232168197632, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.022939106449484825, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.057047583162784576, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05675660818815231, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00045328729902394116, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15616272389888763, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.23632973432540894, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05182572826743126, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9550864100456238, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.3876609802246094, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0131162405014038, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8004726767539978, "metrics/particle/momentum/class_5/event/px_bias": 0.05137212947010994, "metrics/particle/momentum/class_5/event/py_bias": 0.0372336208820343, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5110036730766296, "metrics/particle/momentum/class_5/event/pz_bias": 0.09339258819818497, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.9248415231704712, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3139111995697021, "step_valid_time": 4074.2776906490326, "step_total_time": 4075.5916023254395, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9929626120222319, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09271921601594602, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2887, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530685920577617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540078843626807, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9535380068954195, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045992115637319315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9474893337709223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9484231274638634, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9479560006567066, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530685920577617, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.987917448515489, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0981326312167844, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2154, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9455662862159789, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9468131868131868, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9461893257193059, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05318681318681319, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9411764705882353, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9424175824175824, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9417966176147595, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9455662862159789, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9923716370798659, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0950682455731745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1207, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1205, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9157814871016692, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9206712433257056, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9182198554583492, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07932875667429443, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9142640364188164, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9191456903127384, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9166983643971092, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9157814871016692} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_23000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..bcd1f3d8216e8087fdb53141656e26c75df433b6 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.0894278135657447, "train": {"Classification_binary": 2.8174662590026855, "Classification": 0.019697008654475212, "Regression_pt": 0.14343208074569702, "Regression_eta": 0.009502035565674305, "Regression_sin_phi": 0.006980116944760084, "Regression_cos_phi": 0.006110126152634621, "Regression_energy": 0.14792504906654358, "Total": 8.361337661743164, "Total_unweighted": 3.1511126761324704}, "train_group": {}, "valid": {"Classification_binary": 2.848444620768229, "Classification": 0.020312412331501644, "Regression_pt": 0.1548248032728831, "Regression_eta": 0.012183142205079397, "Regression_sin_phi": 0.007154393941164017, "Regression_cos_phi": 0.007659894103805224, "Regression_energy": 0.1609702706336975, "Total": 8.930109659830729, "Total_unweighted": 3.21154953725636, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020381329581141472, "diagnostic/loss/cld_pf/Classification_binary": 2.8436942100524902, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007375179324299097, "diagnostic/loss/cld_pf/Regression_energy": 0.1599503457546234, "diagnostic/loss/cld_pf/Regression_eta": 0.011617984622716904, "diagnostic/loss/cld_pf/Regression_pt": 0.15480847656726837, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006747423205524683, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08454866707324982, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.208917036652565, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05742938816547394, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4463497996330261, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.046065982431173325, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2028101235628128, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07323312014341354, "diagnostic/regression/cld_pf/pt_residual_rms": 0.43928658962249756, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9309670925140381, "metrics/particle/class_1/pid_accuracy": 0.9839632511138916, "metrics/particle/class_2/efficiency": 0.41403669118881226, "metrics/particle/class_2/pid_accuracy": 0.6929003596305847, "metrics/particle/class_3/efficiency": 0.7633394598960876, "metrics/particle/class_3/pid_accuracy": 0.9818338751792908, "metrics/particle/class_4/efficiency": 0.849646806716919, "metrics/particle/class_4/pid_accuracy": 0.28622329235076904, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.6048387289047241, "metrics/particle/count/bias_mean": 0.3150326907634735, "metrics/particle/count/mae": 3.5830066204071045, "metrics/particle/count/prediction_mean": 70.8803939819336, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.058410659432411194, "metrics/particle/event/energy_response_mean": 0.9942947626113892, "metrics/particle/event/met_abs_error": 6.33751916885376, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04868479073047638, "metrics/particle/event/scalar_pt_response_mean": 1.000449776649475, "metrics/particle/event/vector_pt_closure": 0.05441299453377724, "metrics/particle/matched/delta_eta_abs_mean": 0.011020478792488575, "metrics/particle/matched/delta_phi_abs_mean": 0.01160045899450779, "metrics/particle/matched/delta_r_mean": 0.01792442798614502, "metrics/particle/matched/energy_relative_abs_mean": 0.09470963478088379, "metrics/particle/matched/pid_accuracy": 0.9623596668243408, "metrics/particle/matched/pt_relative_abs_mean": 0.08825436979532242, "metrics/particle/matching/duplicate_fraction": 0.035833172500133514, "metrics/particle/matching/efficiency": 0.817700207233429, "metrics/particle/matching/f1": 0.8158789873123169, "metrics/particle/matching/purity": 0.8140658736228943, "metrics/particle/matching/target_coverage_dr0p05": 0.7928495407104492, "metrics/particle/matching/target_coverage_dr0p10": 0.8562033772468567, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7068864703178406, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8563238382339478, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019468214362859726, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09044470638036728, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08661709725856781, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01645551435649395, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1396302431821823, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0055741360411047935, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3988424837589264, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.024868709966540337, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5878957509994507, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8073.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29147449135780334, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1079901456832886, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1063525676727295, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9834386110305786, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1344946324825287, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.883760452270508, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01782459020614624, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8410829901695251, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06568337231874466, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.021860122680664, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9874.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7963894605636597, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0325381755828857, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.059412002563477, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4893360137939453, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15452685952186584, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.15368929505348206, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.037447929382324, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.4887021780014038, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.042167663574219, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030091822147369385, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09934557229280472, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09866549074649811, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013298098929226398, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15250790119171143, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020012911409139633, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9979841709136963, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12215326726436615, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4616823196411133, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1549.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8170426487922668, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5295720100402832, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21332.8671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21332.6875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28816869854927063, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1079424.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0870753601193428, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0185439586639404, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08320274204015732, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.315496444702148, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2561.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.563477635383606, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.8224379420280457, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.568846702575684, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1924028396606445, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.02336219698190689, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.23151743412017822, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.403997421264648, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.669468879699707, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.137004852294922, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01645175740122795, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0979337990283966, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0904901921749115, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01824011839926243, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1469508558511734, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004070418362971395, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12098227441310883, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0242428220808506, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5021309852600098, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19654.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1637784242630005, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7371609210968018, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5816416144371033, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4615289568901062, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1018715649843216, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.05677318572998, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0039027852471917868, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.29617732763290405, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006505412515252829, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.644883155822754, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22548.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8596422076225281, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9414044618606567, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.018224716186523, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.726194381713867, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5204638242721558, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.5782681703567505, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.562709331512451, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5465929508209229, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.506790637969971, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018005259335041046, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09594333916902542, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08985472470521927, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01748652197420597, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14527244865894318, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002869244432076812, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2440059632062912, "metrics/particle/momentum/all/accepted/pz_bias": -0.02959587797522545, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2241714000701904, "metrics/particle/momentum/all/accepted/samples": 29276.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2476443350315094, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8998380303382874, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1562.402099609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1562.2767333984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12471771985292435, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 292057.625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013921047560870647, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6492739319801331, "metrics/particle/momentum/all/assigned/pz_bias": -0.0288232509046793, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.063232660293579, "metrics/particle/momentum/all/assigned/samples": 34983.0, "metrics/particle/momentum/all/efficiency": 0.8189319968223572, "metrics/particle/momentum/all/event/energy_bias": -1.151504635810852, "metrics/particle/momentum/all/event/p3_error_mean": 16.63254165649414, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.903730392456055, "metrics/particle/momentum/all/event/px_bias": 0.6983529329299927, "metrics/particle/momentum/all/event/py_bias": 0.9634748101234436, "metrics/particle/momentum/all/event/pz_abs_mean": 11.967370986938477, "metrics/particle/momentum/all/event/pz_bias": -1.3658262491226196, "metrics/particle/momentum/all/event/pz_residual_rms": 19.527313232421875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009825468063354492, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0552765317261219, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04439493641257286, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006136463023722172, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09683185070753098, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016956604085862637, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15802574157714844, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.024179477244615555, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9275178909301758, "metrics/particle/momentum/class_1/accepted/samples": 15923.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09313665330410004, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.29961904883384705, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.38303712010383606, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.317375123500824, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0519866980612278, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.170470714569092, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0067098429426550865, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3561168611049652, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.024404171854257584, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3348464965820312, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9288339018821716, "metrics/particle/momentum/class_1/event/energy_bias": 0.6247692108154297, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.576685905456543, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.430110931396484, "metrics/particle/momentum/class_1/event/px_bias": 0.17369207739830017, "metrics/particle/momentum/class_1/event/py_bias": 0.4584275782108307, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.42446517944336, "metrics/particle/momentum/class_1/event/pz_bias": -1.0179375410079956, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.866291046142578, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037308190017938614, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20045891404151917, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20344431698322296, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.041145481169223785, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24329611659049988, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03861955553293228, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.729409098625183, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.25434955954551697, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9294540882110596, "metrics/particle/momentum/class_2/accepted/samples": 1217.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9401707649230957, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7726122140884399, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20946.47265625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20945.966796875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.39953988790512085, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1069653.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08819018304347992, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.2248494625091553, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07525093853473663, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.122750759124756, "metrics/particle/momentum/class_2/assigned/samples": 2608.0, "metrics/particle/momentum/class_2/efficiency": 0.4211072623729706, "metrics/particle/momentum/class_2/event/energy_bias": -7.5164289474487305, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.559765815734863, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.924720287322998, "metrics/particle/momentum/class_2/event/px_bias": -0.20017224550247192, "metrics/particle/momentum/class_2/event/py_bias": 0.47374993562698364, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.426356315612793, "metrics/particle/momentum/class_2/event/pz_bias": -1.5286133289337158, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.703222274780273, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027395883575081825, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14120633900165558, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14043596386909485, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03849191591143608, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1822206974029541, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005998286069370806, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19892264902591705, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.016346478834748268, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9314340949058533, "metrics/particle/momentum/class_3/accepted/samples": 11670.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3078911602497101, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6261793375015259, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5086894035339355, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3781154155731201, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16166973114013672, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.4048490524292, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00931402575224638, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.538641095161438, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.026190537959337234, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6179070472717285, "metrics/particle/momentum/class_3/assigned/samples": 14709.0, "metrics/particle/momentum/class_3/efficiency": 0.7683191895484924, "metrics/particle/momentum/class_3/event/energy_bias": 7.108808517456055, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.186281204223633, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.827925682067871, "metrics/particle/momentum/class_3/event/px_bias": 0.7184152007102966, "metrics/particle/momentum/class_3/event/py_bias": -0.10886477679014206, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.075751781463623, "metrics/particle/momentum/class_3/event/pz_bias": 0.9687182307243347, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.178278923034668, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01384334173053503, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10490308701992035, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10675718635320663, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0463741160929203, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1606559157371521, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010033444501459599, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5650269389152527, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.107968769967556, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0130972862243652, "metrics/particle/momentum/class_4/accepted/samples": 299.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2227679044008255, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.1118457317352295, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.9824559092521667, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.9253512620925903, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12994350492954254, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.776861667633057, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6387980580329895, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.03614766150712967, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.080303430557251, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8446327447891235, "metrics/particle/momentum/class_4/event/energy_bias": -1.3854235410690308, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.21978759765625, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6240638494491577, "metrics/particle/momentum/class_4/event/px_bias": -0.156345397233963, "metrics/particle/momentum/class_4/event/py_bias": 0.06834926456212997, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2169628143310547, "metrics/particle/momentum/class_4/event/pz_bias": 0.2675517797470093, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.9681549072265625, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00849028117954731, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03273342549800873, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03165512531995773, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00737769016996026, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08796817064285278, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19289663434028625, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08032862097024918, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5306567549705505, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04321565479040146, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.09146557003259659, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0803195908665657, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.02852603793144226, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.37878870964050293, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2927779257297516, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.12350258976221085, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0013911724090576, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.016769856214523315, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9086683988571167, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7193160057067871, "metrics/particle/momentum/class_5/event/px_bias": 0.162763312458992, "metrics/particle/momentum/class_5/event/py_bias": 0.07181280106306076, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4419019818305969, "metrics/particle/momentum/class_5/event/pz_bias": -0.055545516312122345, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.742616057395935, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3360595703125, "step_valid_time": 4214.87295794487, "step_total_time": 4216.209017753601, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009710788266482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08811306219829196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2896, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9504430587463079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507550886408405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9505990480879698, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04924491135915955, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9455201837873318, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9458305975049245, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9456753651731495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9504430587463079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007597257527434, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10065201091120335, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2154, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.950812472551603, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9506037321624589, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049187527448397016, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9455662862159789, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9459815546772069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9457738748627882, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9503950834064969, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064198846189212, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10123983418926508, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1230, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392558845861807, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9388994307400379, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060744115413819286, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9332321699544764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9339407744874715, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9335863377609108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385432473444613} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_24000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..f2eb5f22f71d3c994df62746e0e707765e9ddaaa --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.1367942402425162, "train": {"Classification_binary": 2.7859039306640625, "Classification": 0.024127304553985596, "Regression_pt": 0.1588270664215088, "Regression_eta": 0.015218193642795086, "Regression_sin_phi": 0.008867640979588032, "Regression_cos_phi": 0.008370839059352875, "Regression_energy": 0.1665516495704651, "Total": 9.463394165039062, "Total_unweighted": 3.167866624891758}, "train_group": {}, "valid": {"Classification_binary": 2.8809967041015625, "Classification": 0.020372965683539707, "Regression_pt": 0.15623591343561807, "Regression_eta": 0.01242390771706899, "Regression_sin_phi": 0.007325595865646998, "Regression_cos_phi": 0.007684625685214996, "Regression_energy": 0.16194242238998413, "Total": 9.020881652832031, "Total_unweighted": 3.2469821348786354, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020349033176898956, "diagnostic/loss/cld_pf/Classification_binary": 2.8727335929870605, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007434855680912733, "diagnostic/loss/cld_pf/Regression_energy": 0.16098187863826752, "diagnostic/loss/cld_pf/Regression_eta": 0.011766848154366016, "diagnostic/loss/cld_pf/Regression_pt": 0.15608882904052734, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006826821714639664, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08769361674785614, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21079955995082855, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06057433411478996, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4517425000667572, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05259746313095093, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2051340639591217, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07976459711790085, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4457828402519226, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9333052039146423, "metrics/particle/class_1/pid_accuracy": 0.9819294214248657, "metrics/particle/class_2/efficiency": 0.42770877480506897, "metrics/particle/class_2/pid_accuracy": 0.6987213492393494, "metrics/particle/class_3/efficiency": 0.7548567056655884, "metrics/particle/class_3/pid_accuracy": 0.9820373058319092, "metrics/particle/class_4/efficiency": 0.8708375096321106, "metrics/particle/class_4/pid_accuracy": 0.329084575176239, "metrics/particle/class_5/efficiency": 0.9592233300209045, "metrics/particle/class_5/pid_accuracy": 0.6396760940551758, "metrics/particle/count/bias_mean": 0.9039215445518494, "metrics/particle/count/mae": 3.642483711242676, "metrics/particle/count/prediction_mean": 71.46928405761719, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06136495620012283, "metrics/particle/event/energy_response_mean": 1.006649136543274, "metrics/particle/event/met_abs_error": 6.059825420379639, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05343286320567131, "metrics/particle/event/scalar_pt_response_mean": 1.0191311836242676, "metrics/particle/event/vector_pt_closure": 0.055094171315431595, "metrics/particle/matched/delta_eta_abs_mean": 0.010710327886044979, "metrics/particle/matched/delta_phi_abs_mean": 0.01161723118275404, "metrics/particle/matched/delta_r_mean": 0.01762540638446808, "metrics/particle/matched/energy_relative_abs_mean": 0.09412292391061783, "metrics/particle/matched/pid_accuracy": 0.961603045463562, "metrics/particle/matched/pt_relative_abs_mean": 0.08770649880170822, "metrics/particle/matching/duplicate_fraction": 0.04057687520980835, "metrics/particle/matching/efficiency": 0.816542387008667, "metrics/particle/matching/f1": 0.8113458752632141, "metrics/particle/matching/purity": 0.8062150478363037, "metrics/particle/matching/target_coverage_dr0p05": 0.7846153974533081, "metrics/particle/matching/target_coverage_dr0p10": 0.8563145399093628, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7063400149345398, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8558050990104675, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020049134269356728, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08890069276094437, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08619424700737, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014781065285205841, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1394570916891098, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005702243652194738, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.40685558319091797, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.003688666969537735, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6595960855484009, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8067.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.30919966101646423, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.164250373840332, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2334545850753784, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.112328290939331, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13584107160568237, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.688660621643066, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017043162137269974, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8407683372497559, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05325816571712494, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2326369285583496, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9916.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.79579758644104, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.2685723304748535, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.772716522216797, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.924787998199463, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.17315614223480225, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.04294110834598541, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.567698955535889, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.8572896122932434, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.902009010314941, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029223080724477768, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10096292942762375, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10263005644083023, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012577123008668423, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15593650937080383, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023151125758886337, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.087011694908142, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2430553138256073, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.536496162414551, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1555.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8172751665115356, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6601924896240234, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21259.76171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21259.587890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28974854946136475, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1080715.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08355899155139923, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.032665252685547, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1158369928598404, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.211472511291504, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2585.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5656602382659912, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.3551133871078491, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.381831169128418, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1291335821151733, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07658983021974564, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.18501777946949005, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.233774662017822, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.7202043533325195, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.60329818725586, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015604202635586262, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09708654880523682, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08900923281908035, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019712453708052635, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14500807225704193, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00046035804552957416, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1195441409945488, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.002621966414153576, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5013922452926636, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19550.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16861160099506378, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8197377920150757, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7057541608810425, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5923783779144287, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1070290356874466, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.867497444152832, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0035410765558481216, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2889574468135834, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.025525476783514023, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4861916303634644, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22592.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8550933599472046, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.5531768798828125, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.756911277770996, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.132054328918457, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.1297483444213867, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7776356935501099, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.293222904205322, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.49640691280365, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.666958808898926, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017559317871928215, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09502951800823212, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08895684778690338, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017968419939279556, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14410410821437836, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00311942957341671, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25056543946266174, "metrics/particle/momentum/all/accepted/pz_bias": -0.012218841351568699, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.26364004611969, "metrics/particle/momentum/all/accepted/samples": 29172.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25611811876296997, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9789935350418091, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1566.826904296875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1566.70703125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1286296397447586, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 293313.15625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013250505551695824, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6469844579696655, "metrics/particle/momentum/all/assigned/pz_bias": 0.02294876240193844, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0564002990722656, "metrics/particle/momentum/all/assigned/samples": 35093.0, "metrics/particle/momentum/all/efficiency": 0.8160228133201599, "metrics/particle/momentum/all/event/energy_bias": 3.4666359424591064, "metrics/particle/momentum/all/event/p3_error_mean": 16.289207458496094, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.726948738098145, "metrics/particle/momentum/all/event/px_bias": -1.0331820249557495, "metrics/particle/momentum/all/event/py_bias": -0.63555908203125, "metrics/particle/momentum/all/event/pz_abs_mean": 11.746901512145996, "metrics/particle/momentum/all/event/pz_bias": 3.0739006996154785, "metrics/particle/momentum/all/event/pz_residual_rms": 18.71145248413086, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009236425161361694, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05502001941204071, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0432867594063282, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006219516042619944, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09475403279066086, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019417476141825318, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15976518392562866, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.011798844672739506, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9826589226722717, "metrics/particle/momentum/class_1/accepted/samples": 15965.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08692899346351624, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.29583945870399475, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3910009264945984, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3327862620353699, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04987458512187004, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.1339545249938965, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005891617387533188, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.33415907621383667, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005660837981849909, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1632704734802246, "metrics/particle/momentum/class_1/assigned/samples": 17143.0, "metrics/particle/momentum/class_1/efficiency": 0.9312838912010193, "metrics/particle/momentum/class_1/event/energy_bias": 3.455267906188965, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.995627403259277, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.811600685119629, "metrics/particle/momentum/class_1/event/px_bias": 0.034622177481651306, "metrics/particle/momentum/class_1/event/py_bias": -0.2105235904455185, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.65075969696045, "metrics/particle/momentum/class_1/event/pz_bias": 0.38892537355422974, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.604156494140625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03801046684384346, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19943836331367493, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20059707760810852, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03162211552262306, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23981745541095734, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.036565978080034256, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8172465562820435, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17135988175868988, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.339395999908447, "metrics/particle/momentum/class_2/accepted/samples": 1258.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9394451975822449, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.826600968837738, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20565.115234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20564.6328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.38585329055786133, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1062976.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0815868228673935, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1921963691711426, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.019424952566623688, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.259003162384033, "metrics/particle/momentum/class_2/assigned/samples": 2672.0, "metrics/particle/momentum/class_2/efficiency": 0.43529412150382996, "metrics/particle/momentum/class_2/event/energy_bias": -5.829866886138916, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.55975341796875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.958754539489746, "metrics/particle/momentum/class_2/event/px_bias": -0.6097393035888672, "metrics/particle/momentum/class_2/event/py_bias": -0.08940492570400238, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.388555526733398, "metrics/particle/momentum/class_2/event/pz_bias": 1.182910442352295, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.61758804321289, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027179809287190437, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13980388641357422, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14057989418506622, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.038996513932943344, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1822143793106079, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009583550854586065, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19859051704406738, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0008570313220843673, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7340659499168396, "metrics/particle/momentum/class_3/accepted/samples": 11478.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3329325020313263, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.781392216682434, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8356093168258667, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7066450119018555, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17483560740947723, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.40218448638916, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00962646584957838, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.555087149143219, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04868490993976593, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.671189785003662, "metrics/particle/momentum/class_3/assigned/samples": 14751.0, "metrics/particle/momentum/class_3/efficiency": 0.7556784749031067, "metrics/particle/momentum/class_3/event/energy_bias": 6.5539398193359375, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.961137771606445, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.8614912033081055, "metrics/particle/momentum/class_3/event/px_bias": -0.30614155530929565, "metrics/particle/momentum/class_3/event/py_bias": -0.5244764089584351, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.871127605438232, "metrics/particle/momentum/class_3/event/pz_bias": 1.1932741403579712, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.540656089782715, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012442889623343945, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11189410835504532, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11197081208229065, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0500868558883667, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16564612090587616, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009803921915590763, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5581663846969604, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17612361907958984, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0293617248535156, "metrics/particle/momentum/class_4/accepted/samples": 306.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.20166900753974915, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2286083698272705, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.497384548187256, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.438166379928589, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11864406615495682, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.695476531982422, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6151378154754639, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1497303545475006, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.990332007408142, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8644067645072937, "metrics/particle/momentum/class_4/event/energy_bias": -0.9776820540428162, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4420390129089355, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.758521556854248, "metrics/particle/momentum/class_4/event/px_bias": -0.24207991361618042, "metrics/particle/momentum/class_4/event/py_bias": 0.12080472707748413, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3972973823547363, "metrics/particle/momentum/class_4/event/pz_bias": 0.18941156566143036, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.280330181121826, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007189859636127949, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02427055686712265, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022943731397390366, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.010498459450900555, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06061558425426483, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1365242749452591, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.02081502042710781, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.43760955333709717, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02921251393854618, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05388997495174408, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05262913927435875, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.014217100106179714, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.040462426841259, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.16702687740325928, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2354738712310791, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.06332411617040634, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9276506304740906, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": 0.2649770975112915, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9266803860664368, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7321960926055908, "metrics/particle/momentum/class_5/event/px_bias": 0.09015654027462006, "metrics/particle/momentum/class_5/event/py_bias": 0.06804109364748001, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.461306095123291, "metrics/particle/momentum/class_5/event/pz_bias": 0.11937938630580902, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.895917296409607, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.2075605392456055, "step_valid_time": 4354.363127231598, "step_total_time": 4355.570688009262, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0094916844139399, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09757264415037148, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3048, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2893, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9553659337052839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9550524934383202, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9552091878589007, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04494750656167979, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9494584837545126, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9491469816272966, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9493027071369975, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9553659337052839, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0098556916547006, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10353032365574322, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2289, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2150, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2137, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9438103599648815, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392747924858017, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9415371140792643, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06072520751419834, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9381035996488147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9335954565312363, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9358440989708781, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9438103599648815, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011833542734104, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10407093829456193, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1214, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07283763277693475, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.921092564491654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.921092564491654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.921092564491654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9271623672230652} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_25000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..00df0eebb9a2a36407d292897b18bae554b2e939 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1841606669192877, "train": {"Classification_binary": 2.884441375732422, "Classification": 0.019205152988433838, "Regression_pt": 0.1580868363380432, "Regression_eta": 0.012605268508195877, "Regression_sin_phi": 0.007281731814146042, "Regression_cos_phi": 0.009038740769028664, "Regression_energy": 0.1672886461019516, "Total": 9.152618408203125, "Total_unweighted": 3.257947752252221}, "train_group": {}, "valid": {"Classification_binary": 2.845500946044922, "Classification": 0.020017715791861217, "Regression_pt": 0.15370859702428183, "Regression_eta": 0.012242628882328669, "Regression_sin_phi": 0.007228105639417966, "Regression_cos_phi": 0.007639330501357715, "Regression_energy": 0.15996705492337546, "Total": 8.90013567606608, "Total_unweighted": 3.2063043788075447, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019962187856435776, "diagnostic/loss/cld_pf/Classification_binary": 2.837891101837158, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007295849733054638, "diagnostic/loss/cld_pf/Regression_energy": 0.15824972093105316, "diagnostic/loss/cld_pf/Regression_eta": 0.0114214476197958, "diagnostic/loss/cld_pf/Regression_pt": 0.15296465158462524, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0067902179434895515, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08723591268062592, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20946142077445984, "diagnostic/regression/cld_pf/energy_residual_mean": 0.060116637498140335, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4463764727115631, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.053431589156389236, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20515784621238708, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08059871941804886, "diagnostic/regression/cld_pf/pt_residual_rms": 0.44072431325912476, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9269615411758423, "metrics/particle/class_1/pid_accuracy": 0.9828808903694153, "metrics/particle/class_2/efficiency": 0.42668336629867554, "metrics/particle/class_2/pid_accuracy": 0.7158878445625305, "metrics/particle/class_3/efficiency": 0.7587684392929077, "metrics/particle/class_3/pid_accuracy": 0.9803052544593811, "metrics/particle/class_4/efficiency": 0.8193743824958801, "metrics/particle/class_4/pid_accuracy": 0.2857142984867096, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.6211812496185303, "metrics/particle/count/bias_mean": -0.045098040252923965, "metrics/particle/count/mae": 3.4725489616394043, "metrics/particle/count/prediction_mean": 70.520263671875, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.058290280401706696, "metrics/particle/event/energy_response_mean": 0.9958526492118835, "metrics/particle/event/met_abs_error": 6.132661819458008, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04913128539919853, "metrics/particle/event/scalar_pt_response_mean": 1.0132979154586792, "metrics/particle/event/vector_pt_closure": 0.05284229665994644, "metrics/particle/matched/delta_eta_abs_mean": 0.011692080646753311, "metrics/particle/matched/delta_phi_abs_mean": 0.011219522915780544, "metrics/particle/matched/delta_r_mean": 0.018136369064450264, "metrics/particle/matched/energy_relative_abs_mean": 0.09460148960351944, "metrics/particle/matched/pid_accuracy": 0.9620431661605835, "metrics/particle/matched/pt_relative_abs_mean": 0.09005442261695862, "metrics/particle/matching/duplicate_fraction": 0.03858345001935959, "metrics/particle/matching/efficiency": 0.814541757106781, "metrics/particle/matching/f1": 0.8148021101951599, "metrics/particle/matching/purity": 0.8150626420974731, "metrics/particle/matching/target_coverage_dr0p05": 0.7888852953910828, "metrics/particle/matching/target_coverage_dr0p10": 0.855860710144043, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7018200159072876, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.852035403251648, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01988622546195984, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08981535583734512, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08845380693674088, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02368452027440071, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1401047259569168, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005824761465191841, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.40892675518989563, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07601917535066605, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5830903053283691, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8069.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2842664122581482, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9887444376945496, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2100647687911987, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1008100509643555, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13251422345638275, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.063880920410156, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014723801985383034, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7832477688789368, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0964844599366188, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6050074100494385, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9848.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7959948778152466, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.8058671951293945, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.950138092041016, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.894136428833008, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15276727080345154, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.20453408360481262, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.753042221069336, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.450711965560913, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.284488677978516, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030105378478765488, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10277026146650314, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10477341711521149, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.026998313143849373, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15743249654769897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025308242067694664, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0682240724563599, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3324893116950989, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6265995502471924, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1541.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8049216270446777, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4319472312927246, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 20986.505859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 20986.333984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2732992470264435, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1058144.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07668108493089676, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.853411912918091, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2929442226886749, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.396919250488281, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2543.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5605674982070923, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.114946365356445, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.858142852783203, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0887560844421387, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.028017200529575348, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.024430856108665466, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.720952987670898, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.801140546798706, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.9180908203125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016378069296479225, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09671840816736221, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0905812606215477, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02099541388452053, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14527906477451324, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006156371673569083, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12527528405189514, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.019086798653006554, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5332866907119751, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19492.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16877742111682892, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7315021753311157, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6175469160079956, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5035426020622253, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10628470033407211, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.037925720214844, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0038196758832782507, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.29517143964767456, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004006543196737766, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.484512209892273, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22515.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8525565266609192, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5172808170318604, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.07364559173584, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.699053764343262, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.08078213036060333, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.41380372643470764, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.72177791595459, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.747102677822113, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.016074180603027, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01807764731347561, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09512487798929214, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09074287861585617, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022058876231312752, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1445385217666626, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00336746615357697, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2538529634475708, "metrics/particle/momentum/all/accepted/pz_bias": -0.051467414945364, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2576932907104492, "metrics/particle/momentum/all/accepted/samples": 29102.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2477051317691803, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8551070690155029, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1529.6656494140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1529.548828125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1258522868156433, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 285606.625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012204205617308617, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6192470192909241, "metrics/particle/momentum/all/assigned/pz_bias": -0.05114717781543732, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9107983112335205, "metrics/particle/momentum/all/assigned/samples": 34906.0, "metrics/particle/momentum/all/efficiency": 0.8140647411346436, "metrics/particle/momentum/all/event/energy_bias": -0.7917982935905457, "metrics/particle/momentum/all/event/p3_error_mean": 16.673437118530273, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.504454612731934, "metrics/particle/momentum/all/event/px_bias": 0.10000234842300415, "metrics/particle/momentum/all/event/py_bias": 0.18483884632587433, "metrics/particle/momentum/all/event/pz_abs_mean": 12.29493236541748, "metrics/particle/momentum/all/event/pz_bias": -4.504749774932861, "metrics/particle/momentum/all/event/pz_residual_rms": 21.34527015686035, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009585080668330193, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.054014772176742554, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04610423743724823, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01557181030511856, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09574639797210693, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015782827977091074, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1617194563150406, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03472542762756348, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9092369079589844, "metrics/particle/momentum/class_1/accepted/samples": 15840.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09671467542648315, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3235648274421692, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.46529099345207214, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4099562168121338, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05642431974411011, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.34880256652832, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005776636768132448, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.36382773518562317, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04730399698019028, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.581890821456909, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9239922761917114, "metrics/particle/momentum/class_1/event/energy_bias": 4.429679870605469, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.74486255645752, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.5044121742248535, "metrics/particle/momentum/class_1/event/px_bias": 0.8414840698242188, "metrics/particle/momentum/class_1/event/py_bias": 0.18473777174949646, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.407241821289062, "metrics/particle/momentum/class_1/event/pz_bias": -1.195334792137146, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.618408203125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03767922520637512, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19766272604465485, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19940705597400665, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03698717802762985, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24031274020671844, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04711616411805153, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8603408336639404, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.29083508253097534, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.366002082824707, "metrics/particle/momentum/class_2/accepted/samples": 1231.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9242888689041138, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7265583276748657, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20627.4375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20626.96875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37611132860183716, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1049107.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07962891459465027, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9935128688812256, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1264367550611496, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.249072074890137, "metrics/particle/momentum/class_2/assigned/samples": 2587.0, "metrics/particle/momentum/class_2/efficiency": 0.4259515702724457, "metrics/particle/momentum/class_2/event/energy_bias": -8.66667366027832, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.187198638916016, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.825815200805664, "metrics/particle/momentum/class_2/event/px_bias": 0.0025920122861862183, "metrics/particle/momentum/class_2/event/py_bias": 0.5614848136901855, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.052295684814453, "metrics/particle/momentum/class_2/event/pz_bias": -1.5877169370651245, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.188867568969727, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027868298813700676, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14076592028141022, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14038051664829254, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0366407185792923, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18175742030143738, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011220438173040748, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2018505036830902, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.049645766615867615, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.857275128364563, "metrics/particle/momentum/class_3/accepted/samples": 11586.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3058491349220276, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4544917345046997, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.515519618988037, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3867439031600952, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16261771321296692, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.296717643737793, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007984167896211147, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4969269633293152, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04390178993344307, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.285501003265381, "metrics/particle/momentum/class_3/assigned/samples": 14654.0, "metrics/particle/momentum/class_3/efficiency": 0.7627888321876526, "metrics/particle/momentum/class_3/event/energy_bias": 4.621954441070557, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.710442543029785, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.546962738037109, "metrics/particle/momentum/class_3/event/px_bias": -0.6111677885055542, "metrics/particle/momentum/class_3/event/py_bias": -0.5319007039070129, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.769025802612305, "metrics/particle/momentum/class_3/event/pz_bias": -1.902307152748108, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.486310958862305, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012912473641335964, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12052951753139496, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12085890769958496, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.054713211953639984, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17090623080730438, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007117437664419413, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5987001657485962, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.03757356107234955, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8105690479278564, "metrics/particle/momentum/class_4/accepted/samples": 281.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3005020320415497, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.104262590408325, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.2076401710510254, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.1380553245544434, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17514124512672424, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.32506561279297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8641655445098877, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.03135613724589348, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2707455158233643, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7937853336334229, "metrics/particle/momentum/class_4/event/energy_bias": -1.2667592763900757, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4424824714660645, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8273659944534302, "metrics/particle/momentum/class_4/event/px_bias": -0.26860886812210083, "metrics/particle/momentum/class_4/event/py_bias": -0.04517683386802673, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.26620614528656, "metrics/particle/momentum/class_4/event/pz_bias": 0.06471860408782959, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.913547515869141, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008380117826163769, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.0281996950507164, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02822231687605381, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005715548060834408, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06622815132141113, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17706499993801117, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.024279968813061714, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.47473883628845215, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.054756660014390945, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06055456027388573, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06092649698257446, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00046156803728081286, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17165683209896088, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2777792513370514, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08854833990335464, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9247552752494812, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": 0.09000077098608017, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9506640434265137, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7614495158195496, "metrics/particle/momentum/class_5/event/px_bias": 0.13570290803909302, "metrics/particle/momentum/class_5/event/py_bias": 0.015693724155426025, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.46135804057121277, "metrics/particle/momentum/class_5/event/pz_bias": 0.1158904880285263, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7162197828292847, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3252978324890137, "step_valid_time": 4494.593561410904, "step_total_time": 4495.91885972023, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0115086747645516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09682648905949365, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3049, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.95274040039383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9521154476877665, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524278215223098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04788455231223352, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9458483754512635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9452279435880616, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9455380577427821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.95274040039383, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071440857316876, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10130051877818702, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2282, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2158, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9473222124670764, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9456617002629273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9464912280701754, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05433829973707274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9411764705882353, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9395267309377738, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9403508771929825, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9473222124670764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0118887683867146, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10276206176964986, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1218, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1214, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9241274658573596, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9234268385140257, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9237770193401593, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07657316148597422, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.921092564491654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9203942380591357, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9207432688661358, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9241274658573596} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_26000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..1c69c72c4b3915bd4a681c99a292fc29013c16e7 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2315270935960592, "train": {"Classification_binary": 2.7321715354919434, "Classification": 0.022160187363624573, "Regression_pt": 0.13983751833438873, "Regression_eta": 0.013098499737679958, "Regression_sin_phi": 0.00650195125490427, "Regression_cos_phi": 0.007265576161444187, "Regression_energy": 0.148168683052063, "Total": 8.518841743469238, "Total_unweighted": 3.069203951396048}, "train_group": {}, "valid": {"Classification_binary": 2.780226707458496, "Classification": 0.019626652201016743, "Regression_pt": 0.14979527393976846, "Regression_eta": 0.01197606697678566, "Regression_sin_phi": 0.007008599738279979, "Regression_cos_phi": 0.007574933891495069, "Regression_energy": 0.15619495511054993, "Total": 8.695560455322266, "Total_unweighted": 3.132403189316392, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019721437245607376, "diagnostic/loss/cld_pf/Classification_binary": 2.774383544921875, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00727761909365654, "diagnostic/loss/cld_pf/Regression_energy": 0.1557719111442566, "diagnostic/loss/cld_pf/Regression_eta": 0.01126603502780199, "diagnostic/loss/cld_pf/Regression_pt": 0.15014389157295227, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006637921556830406, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07195408642292023, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20456112921237946, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04483480378985405, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4402938485145569, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.038969554007053375, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19965402781963348, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06613669544458389, "diagnostic/regression/cld_pf/pt_residual_rms": 0.43433576822280884, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9280923008918762, "metrics/particle/class_1/pid_accuracy": 0.9851112961769104, "metrics/particle/class_2/efficiency": 0.4211006164550781, "metrics/particle/class_2/pid_accuracy": 0.7150974273681641, "metrics/particle/class_3/efficiency": 0.7697345018386841, "metrics/particle/class_3/pid_accuracy": 0.984611451625824, "metrics/particle/class_4/efficiency": 0.8203834295272827, "metrics/particle/class_4/pid_accuracy": 0.3050430417060852, "metrics/particle/class_5/efficiency": 0.955339789390564, "metrics/particle/class_5/pid_accuracy": 0.6280487775802612, "metrics/particle/count/bias_mean": -0.8915032744407654, "metrics/particle/count/mae": 3.5424835681915283, "metrics/particle/count/prediction_mean": 69.67385864257812, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05715698376297951, "metrics/particle/event/energy_response_mean": 0.9849893450737, "metrics/particle/event/met_abs_error": 5.825920581817627, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04685472324490547, "metrics/particle/event/scalar_pt_response_mean": 0.9941838979721069, "metrics/particle/event/vector_pt_closure": 0.051374271512031555, "metrics/particle/matched/delta_eta_abs_mean": 0.010317318141460419, "metrics/particle/matched/delta_phi_abs_mean": 0.010258069261908531, "metrics/particle/matched/delta_r_mean": 0.016320357099175453, "metrics/particle/matched/energy_relative_abs_mean": 0.09246259182691574, "metrics/particle/matched/pid_accuracy": 0.9653940796852112, "metrics/particle/matched/pt_relative_abs_mean": 0.08682075142860413, "metrics/particle/matching/duplicate_fraction": 0.03172578290104866, "metrics/particle/matching/efficiency": 0.8192747831344604, "metrics/particle/matching/f1": 0.8244829177856445, "metrics/particle/matching/purity": 0.8297576904296875, "metrics/particle/matching/target_coverage_dr0p05": 0.7964062690734863, "metrics/particle/matching/target_coverage_dr0p10": 0.8538044691085815, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7098689675331116, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8563793897628784, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018834494054317474, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08701138943433762, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08513498306274414, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011984202079474926, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1367502510547638, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0049683270044624805, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3782453238964081, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0063979472033679485, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5175074338912964, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8051.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2839343845844269, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0393633842468262, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0334808826446533, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9098403453826904, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1278534084558487, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.32493782043457, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016071245074272156, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7959992289543152, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.00927188340574503, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.974827766418457, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9769.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7942191958427429, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.1218196153640747, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.317106246948242, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6495323181152344, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.03565796837210655, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.09194324165582657, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.194718360900879, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5417683720588684, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.952362060546875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027477258816361427, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09603496640920639, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09706112742424011, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018920743837952614, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.148961141705513, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0229357797652483, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9814830422401428, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2018144279718399, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.165889024734497, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1526.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8021047711372375, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.608290672302246, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 18599.146484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 18598.974609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2756410241127014, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 929068.3125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08413461595773697, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.169853687286377, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0630045086145401, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.566363334655762, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2496.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5551109313964844, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.5059195756912231, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.879605293273926, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1958861351013184, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.013648688793182373, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04527759924530983, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.726703643798828, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5287479162216187, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.240896224975586, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014205040410161018, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09582238644361496, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0886242613196373, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01166758593171835, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14455223083496094, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005582056473940611, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11595913767814636, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00810229778289795, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5093066692352295, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19706.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.152060866355896, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7155197262763977, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5520851612091064, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42944246530532837, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09323509782552719, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.446254730224609, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0032596562523394823, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2724655270576477, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012857752852141857, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3350473642349243, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22395.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8619166612625122, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.9560080766677856, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.863421440124512, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.561957359313965, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.06346269696950912, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7651598453521729, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6255850791931152, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.36287185549736023, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.552208423614502, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01616949588060379, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09341099113225937, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08810458332300186, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01213261391967535, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14268629252910614, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002936857519671321, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2331758290529251, "metrics/particle/momentum/all/accepted/pz_bias": -0.006823577452450991, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1532586812973022, "metrics/particle/momentum/all/accepted/samples": 29283.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2360418736934662, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8710876107215881, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1340.044189453125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1339.917724609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1161281019449234, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 249319.15625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012694749049842358, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6286767721176147, "metrics/particle/momentum/all/assigned/pz_bias": 0.01545832771807909, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1994199752807617, "metrics/particle/momentum/all/assigned/samples": 34660.0, "metrics/particle/momentum/all/efficiency": 0.8191277980804443, "metrics/particle/momentum/all/event/energy_bias": -3.583747148513794, "metrics/particle/momentum/all/event/p3_error_mean": 15.984132766723633, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.267036437988281, "metrics/particle/momentum/all/event/px_bias": 0.11276932805776596, "metrics/particle/momentum/all/event/py_bias": -0.8118255138397217, "metrics/particle/momentum/all/event/pz_abs_mean": 11.68188190460205, "metrics/particle/momentum/all/event/pz_bias": 1.4333882331848145, "metrics/particle/momentum/all/event/pz_residual_rms": 20.162696838378906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009199019521474838, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05339258164167404, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04316530004143715, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00893376674503088, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0950004979968071, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013865254586562514, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14321863651275635, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0016635969514027238, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.873083770275116, "metrics/particle/momentum/class_1/accepted/samples": 15867.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09934882074594498, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3191956877708435, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3670814335346222, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.30356335639953613, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.053116973489522934, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.594676494598389, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006479103583842516, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.35387954115867615, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.016360674053430557, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3036322593688965, "metrics/particle/momentum/class_1/assigned/samples": 17132.0, "metrics/particle/momentum/class_1/efficiency": 0.9255672693252563, "metrics/particle/momentum/class_1/event/energy_bias": 1.3721590042114258, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.235880851745605, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.043208122253418, "metrics/particle/momentum/class_1/event/px_bias": 0.5397202968597412, "metrics/particle/momentum/class_1/event/py_bias": -0.2379244714975357, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.209781646728516, "metrics/particle/momentum/class_1/event/pz_bias": -0.11087873578071594, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.483736038208008, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03639217093586922, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19842739403247833, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19969917833805084, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.044463861733675, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2379959523677826, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04098360612988472, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7817350625991821, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.20910952985286713, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.037472724914551, "metrics/particle/momentum/class_2/accepted/samples": 1220.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8856838345527649, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7320433259010315, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 18361.86328125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 18361.37109375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3829113841056824, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 923169.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07594936341047287, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1697819232940674, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07355328649282455, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.7330851554870605, "metrics/particle/momentum/class_2/assigned/samples": 2528.0, "metrics/particle/momentum/class_2/efficiency": 0.4221453368663788, "metrics/particle/momentum/class_2/event/energy_bias": -8.880818367004395, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.870731353759766, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.467024803161621, "metrics/particle/momentum/class_2/event/px_bias": -0.4348752796649933, "metrics/particle/momentum/class_2/event/py_bias": 0.008677393198013306, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.954146385192871, "metrics/particle/momentum/class_2/event/pz_bias": 0.00902700424194336, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.211782455444336, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023695193231105804, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13686051964759827, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13728775084018707, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02145378664135933, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17841199040412903, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000936489028390497, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18483982980251312, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004389231558889151, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6856858134269714, "metrics/particle/momentum/class_3/accepted/samples": 11746.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2856890857219696, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5410184860229492, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4004325866699219, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2602890729904175, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14371588826179504, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.369858741760254, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009258619509637356, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.513626754283905, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0254813265055418, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7693071365356445, "metrics/particle/momentum/class_3/assigned/samples": 14473.0, "metrics/particle/momentum/class_3/efficiency": 0.7733228206634521, "metrics/particle/momentum/class_3/event/energy_bias": 5.538880348205566, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.626289367675781, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.54868221282959, "metrics/particle/momentum/class_3/event/px_bias": 0.13312841951847076, "metrics/particle/momentum/class_3/event/py_bias": -0.7602965831756592, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.687137126922607, "metrics/particle/momentum/class_3/event/pz_bias": 0.8101757764816284, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.496590614318848, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0124475983902812, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12276246398687363, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12389817088842392, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05029621720314026, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18200939893722534, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01056338008493185, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6582368612289429, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.24174396693706512, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.225752830505371, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28396472334861755, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5869333744049072, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6300314664840698, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5615949630737305, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17231638729572296, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.251181602478027, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6811812520027161, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.21046577394008636, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.111013889312744, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.7295116186141968, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0897908210754395, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5868401527404785, "metrics/particle/momentum/class_4/event/px_bias": -0.21847598254680634, "metrics/particle/momentum/class_4/event/py_bias": 0.05245848000049591, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1031432151794434, "metrics/particle/momentum/class_4/event/pz_bias": 0.42158904671669006, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.131838798522949, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007669822312891483, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022072935476899147, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02205267921090126, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008993078023195267, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.059408582746982574, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1436903029680252, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.07110327482223511, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4565265476703644, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.028069129213690758, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04564785584807396, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0460275262594223, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008713270537555218, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1396642029285431, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.22662509977817535, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.010744939558207989, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8572889566421509, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.11554262787103653, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.982270359992981, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7244149446487427, "metrics/particle/momentum/class_5/event/px_bias": 0.09327182918787003, "metrics/particle/momentum/class_5/event/py_bias": 0.125259667634964, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5135504603385925, "metrics/particle/momentum/class_5/event/pz_bias": 0.30347514152526855, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.9264049530029297, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3371913433074951, "step_valid_time": 4635.118403673172, "step_total_time": 4636.455595254898, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000629202073652, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0834338763781084, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2920, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9583196586806695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9646514701024116, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.961475139940731, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03534852989758837, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9553659337052839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9616782292699042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9585116891669411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9583196586806695, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004381342206479, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09289472274184185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2158, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2151, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9473222124670764, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9502421840598855, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9487799516377226, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04975781594011449, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9442493415276558, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9471598414795245, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9457023521653111, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9473222124670764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008603486086003, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09313768483675444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1230, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9332321699544764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9367859862909368, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.935005701254276, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06321401370906321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9294385432473444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9329779131759329, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9312048650703155, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9332321699544764} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_27000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..11d4c4e8d788ad3ab724d76874c8f76ccfe6cb99 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2788935202728307, "train": {"Classification_binary": 2.784262180328369, "Classification": 0.019512740895152092, "Regression_pt": 0.1456938534975052, "Regression_eta": 0.010020741261541843, "Regression_sin_phi": 0.006816497538238764, "Regression_cos_phi": 0.00852158758789301, "Regression_energy": 0.14840766787528992, "Total": 8.517485618591309, "Total_unweighted": 3.12323526898399}, "train_group": {}, "valid": {"Classification_binary": 2.8135293324788413, "Classification": 0.019861345489819843, "Regression_pt": 0.1535646915435791, "Regression_eta": 0.012750929842392603, "Regression_sin_phi": 0.007192408666014671, "Regression_cos_phi": 0.007596895098686218, "Regression_energy": 0.15936229626337686, "Total": 8.87049420674642, "Total_unweighted": 3.1738578993827105, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019722625613212585, "diagnostic/loss/cld_pf/Classification_binary": 2.8074562549591064, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007209212519228458, "diagnostic/loss/cld_pf/Regression_energy": 0.1576327532529831, "diagnostic/loss/cld_pf/Regression_eta": 0.011552685871720314, "diagnostic/loss/cld_pf/Regression_pt": 0.1527264565229416, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006805134005844593, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0844837874174118, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21032783389091492, "diagnostic/regression/cld_pf/energy_residual_mean": 0.057364508509635925, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4493522644042969, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04503921419382095, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20440304279327393, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07220634818077087, "diagnostic/regression/cld_pf/pt_residual_rms": 0.44321706891059875, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9282647967338562, "metrics/particle/class_1/pid_accuracy": 0.9831113815307617, "metrics/particle/class_2/efficiency": 0.42770877480506897, "metrics/particle/class_2/pid_accuracy": 0.7352157831192017, "metrics/particle/class_3/efficiency": 0.7678665518760681, "metrics/particle/class_3/pid_accuracy": 0.9788214564323425, "metrics/particle/class_4/efficiency": 0.8435923457145691, "metrics/particle/class_4/pid_accuracy": 0.31578946113586426, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.6272912621498108, "metrics/particle/count/bias_mean": 0.17908497154712677, "metrics/particle/count/mae": 3.437908411026001, "metrics/particle/count/prediction_mean": 70.74444580078125, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05890108644962311, "metrics/particle/event/energy_response_mean": 1.0141390562057495, "metrics/particle/event/met_abs_error": 6.5733113288879395, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05028286203742027, "metrics/particle/event/scalar_pt_response_mean": 1.0168496370315552, "metrics/particle/event/vector_pt_closure": 0.05398905649781227, "metrics/particle/matched/delta_eta_abs_mean": 0.010780680924654007, "metrics/particle/matched/delta_phi_abs_mean": 0.010847733356058598, "metrics/particle/matched/delta_r_mean": 0.01716029830276966, "metrics/particle/matched/energy_relative_abs_mean": 0.09435664862394333, "metrics/particle/matched/pid_accuracy": 0.9626145958900452, "metrics/particle/matched/pt_relative_abs_mean": 0.08772315829992294, "metrics/particle/matching/duplicate_fraction": 0.03926495835185051, "metrics/particle/matching/efficiency": 0.819311797618866, "metrics/particle/matching/f1": 0.8182734847068787, "metrics/particle/matching/purity": 0.8172377943992615, "metrics/particle/matching/target_coverage_dr0p05": 0.7900430560112, "metrics/particle/matching/target_coverage_dr0p10": 0.8561107516288757, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7104339599609375, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8575001358985901, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020643820986151695, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09032607078552246, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08699481934309006, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00962951872497797, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1386823207139969, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005615234375, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.399089515209198, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0024879947304725647, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6260814666748047, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8192.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2760712802410126, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.225190281867981, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1593266725540161, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0408059358596802, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12584587931632996, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.653661727905273, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016765983775258064, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8471009731292725, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04077550768852234, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.355257749557495, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9901.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8081286549568176, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.8522591590881348, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.070537567138672, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.9030535221099854, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.36485108733177185, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.19686266779899597, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.883365631103516, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.037265896797180176, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.283268928527832, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.031243164092302322, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09621312469244003, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09700779616832733, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016955280676484108, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14729805290699005, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02330096997320652, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9746291041374207, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.10564661771059036, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.367506504058838, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1545.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8040634393692017, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9046050310134888, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 19588.78515625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 19588.611328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28466588258743286, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 996534.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09231363236904144, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1551833152770996, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19634419679641724, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.548521041870117, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2589.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5620225667953491, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.5327880382537842, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.023541450500488, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2235779762268066, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.1509111523628235, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.18001842498779297, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.863792419433594, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2390806674957275, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.34801959991455, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014555125497281551, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09706564247608185, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08873935788869858, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013348089531064034, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14416983723640442, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004607351147569716, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1214824840426445, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005817157682031393, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5285797715187073, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19534.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.167149156332016, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.850195050239563, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.674344003200531, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5543095469474792, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1038273274898529, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.675032615661621, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004094347823411226, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3022829294204712, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.012359279207885265, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.549428105354309, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22470.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8543935418128967, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.3915362358093262, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.260046005249023, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.945725440979004, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.6500670909881592, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5627615451812744, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6934993267059326, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5653666257858276, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.721643447875977, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017139991745352745, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09513446688652039, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08868754655122757, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012497779913246632, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14282391965389252, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031088790856301785, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2442069798707962, "metrics/particle/momentum/all/accepted/pz_bias": -0.010154694318771362, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2349188327789307, "metrics/particle/momentum/all/accepted/samples": 29271.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24516430497169495, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.034482717514038, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1451.4300537109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1451.306640625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12345537543296814, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 271189.28125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014216247014701366, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6678551435470581, "metrics/particle/momentum/all/assigned/pz_bias": -0.03403221815824509, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1848301887512207, "metrics/particle/momentum/all/assigned/samples": 34960.0, "metrics/particle/momentum/all/efficiency": 0.8187921047210693, "metrics/particle/momentum/all/event/energy_bias": 5.776583671569824, "metrics/particle/momentum/all/event/p3_error_mean": 16.640487670898438, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.886404037475586, "metrics/particle/momentum/all/event/px_bias": 2.1658291816711426, "metrics/particle/momentum/all/event/py_bias": -0.579605758190155, "metrics/particle/momentum/all/event/pz_abs_mean": 12.010311126708984, "metrics/particle/momentum/all/event/pz_bias": -1.8417131900787354, "metrics/particle/momentum/all/event/pz_residual_rms": 18.479578018188477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009599589742720127, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.054090291261672974, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04297950491309166, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011293222196400166, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09330753982067108, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016398612642660737, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15524078905582428, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006035249214619398, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9896121025085449, "metrics/particle/momentum/class_1/accepted/samples": 15855.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09386613965034485, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4044893682003021, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4599103629589081, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4036095142364502, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05536756291985512, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.13990306854248, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0069428239949047565, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.36477112770080566, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0467001311480999, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1925809383392334, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9248672723770142, "metrics/particle/momentum/class_1/event/energy_bias": 4.997936248779297, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.017938613891602, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.643368721008301, "metrics/particle/momentum/class_1/event/px_bias": 1.1376895904541016, "metrics/particle/momentum/class_1/event/py_bias": 0.21140730381011963, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.772235870361328, "metrics/particle/momentum/class_1/event/pz_bias": -1.547898292541504, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.972257614135742, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036818522959947586, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1909710317850113, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1950724869966507, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04323706030845642, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23438601195812225, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04006541147828102, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.766869306564331, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.28362947702407837, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.1684956550598145, "metrics/particle/momentum/class_2/accepted/samples": 1223.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9247268438339233, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8637164235115051, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 19428.6796875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 19428.212890625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.39118772745132446, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 992517.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08812260627746582, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.3437001705169678, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19155332446098328, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.15914249420166, "metrics/particle/momentum/class_2/assigned/samples": 2610.0, "metrics/particle/momentum/class_2/efficiency": 0.4231833815574646, "metrics/particle/momentum/class_2/event/energy_bias": -3.406233310699463, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.343924522399902, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.907853126525879, "metrics/particle/momentum/class_2/event/px_bias": 1.0047998428344727, "metrics/particle/momentum/class_2/event/py_bias": -0.7940014600753784, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.074417114257812, "metrics/particle/momentum/class_2/event/pz_bias": -1.8932924270629883, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.190608978271484, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025507552549242973, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1407482624053955, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13924847543239594, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.018395790830254555, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18025578558444977, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00110826944001019, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19895729422569275, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.009320741519331932, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7596365809440613, "metrics/particle/momentum/class_3/accepted/samples": 11730.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3038567900657654, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8002837896347046, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6816916465759277, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5381802320480347, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15575836598873138, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.88703727722168, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009807259775698185, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5510575175285339, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.00456843338906765, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.56805419921875, "metrics/particle/momentum/class_3/assigned/samples": 14683.0, "metrics/particle/momentum/class_3/efficiency": 0.7722694277763367, "metrics/particle/momentum/class_3/event/energy_bias": 5.428081512451172, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.499787330627441, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.675047874450684, "metrics/particle/momentum/class_3/event/px_bias": 0.36045461893081665, "metrics/particle/momentum/class_3/event/py_bias": -0.1271902620792389, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.396223068237305, "metrics/particle/momentum/class_3/event/pz_bias": 0.9765703678131104, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.817904472351074, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013542698696255684, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12909278273582458, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12969648838043213, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0749339833855629, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18407779932022095, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010067113675177097, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5724381804466248, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11668649315834045, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8906370401382446, "metrics/particle/momentum/class_4/accepted/samples": 298.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2249576598405838, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5123438835144043, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.562093734741211, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5093762874603271, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14689265191555023, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.458627700805664, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6761690974235535, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1499851942062378, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.052696943283081, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8418079018592834, "metrics/particle/momentum/class_4/event/energy_bias": -1.302272081375122, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.175889730453491, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.694370985031128, "metrics/particle/momentum/class_4/event/px_bias": -0.4505847692489624, "metrics/particle/momentum/class_4/event/py_bias": 0.0914207398891449, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0559256076812744, "metrics/particle/momentum/class_4/event/pz_bias": 0.5059044361114502, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.6986987590789795, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007484045810997486, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024696974083781242, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.0237899087369442, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009299647063016891, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06193706765770912, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.130925714969635, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.00742287328466773, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.40016812086105347, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.042639996856451035, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.053938351571559906, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05382503196597099, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0015193751314654946, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.040462426841259, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.16203980147838593, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.22218362987041473, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05517417564988136, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.862906277179718, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": 0.0590711310505867, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0128793716430664, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8045274019241333, "metrics/particle/momentum/class_5/event/px_bias": 0.11347004026174545, "metrics/particle/momentum/class_5/event/py_bias": 0.038757894188165665, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5025318264961243, "metrics/particle/momentum/class_5/event/pz_bias": 0.1170026957988739, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.1316192150115967, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.222097635269165, "step_valid_time": 4775.129846096039, "step_total_time": 4776.351944208145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0056710093443577, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09545300116827793, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2910, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2896, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9550377420413522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9613478691774033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9581824168587422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03865213082259663, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9504430587463079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9567228278823918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9535726045439579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9550377420413522, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007962754623752, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10219187479329472, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2162, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9490781387181738, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507475813544415, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9499121265377856, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04925241864555849, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9433713784021072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9450307827616534, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9442003514938488, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9490781387181738, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0090993345660493, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10449838753966834, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9279210925644916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9314546839299315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9296845305967313, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06854531607006854, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9218512898330804, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9253617669459253, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9236031927023945, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9279210925644916} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_28000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..12e1f6507c60fbec6733b563b537d8e9b311cde4 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3262599469496021, "train": {"Classification_binary": 2.7582812309265137, "Classification": 0.020848410204052925, "Regression_pt": 0.152155801653862, "Regression_eta": 0.010689131915569305, "Regression_sin_phi": 0.006780985742807388, "Regression_cos_phi": 0.007007812149822712, "Regression_energy": 0.15775614976882935, "Total": 8.648982048034668, "Total_unweighted": 3.1135195223614573}, "train_group": {}, "valid": {"Classification_binary": 2.7859538396199546, "Classification": 0.019604582339525223, "Regression_pt": 0.15086091558138529, "Regression_eta": 0.012207582592964172, "Regression_sin_phi": 0.007183892031510671, "Regression_cos_phi": 0.007623403022686641, "Regression_energy": 0.15674678484598795, "Total": 8.749213536580404, "Total_unweighted": 3.1401810000340147, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019552402198314667, "diagnostic/loss/cld_pf/Classification_binary": 2.7856783866882324, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0072670127265155315, "diagnostic/loss/cld_pf/Regression_energy": 0.1564573496580124, "diagnostic/loss/cld_pf/Regression_eta": 0.011447545140981674, "diagnostic/loss/cld_pf/Regression_pt": 0.15159407258033752, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006666942965239286, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07286477088928223, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20478861033916473, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04574549198150635, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4381086826324463, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03940030559897423, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19921094179153442, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06656744331121445, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4350297152996063, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9350492358207703, "metrics/particle/class_1/pid_accuracy": 0.9843817353248596, "metrics/particle/class_2/efficiency": 0.4273669719696045, "metrics/particle/class_2/pid_accuracy": 0.7155424952507019, "metrics/particle/class_3/efficiency": 0.7666578888893127, "metrics/particle/class_3/pid_accuracy": 0.9815971851348877, "metrics/particle/class_4/efficiency": 0.8627648949623108, "metrics/particle/class_4/pid_accuracy": 0.28187134861946106, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6242424249649048, "metrics/particle/count/bias_mean": -0.4143790900707245, "metrics/particle/count/mae": 3.3921568393707275, "metrics/particle/count/prediction_mean": 70.1509780883789, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0562172494828701, "metrics/particle/event/energy_response_mean": 0.9873670935630798, "metrics/particle/event/met_abs_error": 5.845168113708496, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045618701726198196, "metrics/particle/event/scalar_pt_response_mean": 1.0051039457321167, "metrics/particle/event/vector_pt_closure": 0.05127885565161705, "metrics/particle/matched/delta_eta_abs_mean": 0.012184375897049904, "metrics/particle/matched/delta_phi_abs_mean": 0.011555143631994724, "metrics/particle/matched/delta_r_mean": 0.01876392960548401, "metrics/particle/matched/energy_relative_abs_mean": 0.09347055107355118, "metrics/particle/matched/pid_accuracy": 0.9631544351577759, "metrics/particle/matched/pt_relative_abs_mean": 0.08579134196043015, "metrics/particle/matching/duplicate_fraction": 0.036699555814266205, "metrics/particle/matching/efficiency": 0.8222664594650269, "metrics/particle/matching/f1": 0.8246878981590271, "metrics/particle/matching/purity": 0.8271235823631287, "metrics/particle/matching/target_coverage_dr0p05": 0.7794285416603088, "metrics/particle/matching/target_coverage_dr0p10": 0.8574260473251343, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7145278453826904, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8585467338562012, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02359517104923725, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0881485790014267, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08334264159202576, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013727737590670586, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1357847899198532, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005059230141341686, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3791445791721344, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07620780915021896, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.546948790550232, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8104.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28485825657844543, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1021190881729126, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2424554824829102, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.12941312789917, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12715056538581848, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.188873291015625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015779294073581696, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7690632939338684, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05270826071500778, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.755077362060547, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9823.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7994475960731506, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.1761177778244019, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.815621376037598, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.51597261428833, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.047259166836738586, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2342987209558487, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.655370712280273, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.534171462059021, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.936068534851074, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03098996728658676, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10423853993415833, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10047188401222229, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01082673016935587, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1546359360218048, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02075471729040146, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9078550338745117, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.23801350593566895, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.788472890853882, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1590.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7843868732452393, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.413759469985962, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 36937.828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 36937.65234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26278239488601685, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1855195.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07887435704469681, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8672070503234863, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.41075292229652405, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.72585391998291, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2523.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5783921480178833, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.422028541564941, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.091897010803223, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1595932245254517, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05534256994724274, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.072874516248703, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.936041355133057, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.7504897117614746, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.183860778808594, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015735505148768425, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.095644511282444, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08710769563913345, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011255629360675812, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14220096170902252, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006607705727219582, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12172858417034149, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02100568823516369, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5369586944580078, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19674.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1577206403017044, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6525294184684753, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.544586181640625, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4278334379196167, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09774436056613922, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.775751113891602, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003737153485417366, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2785114049911499, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0006308616721071303, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3726601600646973, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22477.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8605169653892517, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5436972379684448, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.8662872314453125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.570450782775879, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4565436542034149, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.011529996991157532, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6535675525665283, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.9854843020439148, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.849284648895264, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01873023808002472, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09404131025075912, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08679228276014328, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011914577335119247, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14116930961608887, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002962408121675253, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2353229820728302, "metrics/particle/momentum/all/accepted/pz_bias": -0.047987449914216995, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1289290189743042, "metrics/particle/momentum/all/accepted/samples": 29368.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23898738622665405, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8345040082931519, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2676.926025390625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2676.805908203125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11799672991037369, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 499361.5625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012577894143760204, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6044446229934692, "metrics/particle/momentum/all/assigned/pz_bias": -0.04422086104750633, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9789035320281982, "metrics/particle/momentum/all/assigned/samples": 34823.0, "metrics/particle/momentum/all/efficiency": 0.8215054869651794, "metrics/particle/momentum/all/event/energy_bias": -2.702213764190674, "metrics/particle/momentum/all/event/p3_error_mean": 15.274849891662598, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.008771896362305, "metrics/particle/momentum/all/event/px_bias": 0.4646270275115967, "metrics/particle/momentum/all/event/py_bias": -0.14989422261714935, "metrics/particle/momentum/all/event/pz_abs_mean": 11.130327224731445, "metrics/particle/momentum/all/event/pz_bias": -2.2991771697998047, "metrics/particle/momentum/all/event/pz_residual_rms": 19.106449127197266, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009224457666277885, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05397536978125572, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04165156930685043, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004008897580206394, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09238352626562119, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002064693719148636, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15332379937171936, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02420308068394661, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9444646835327148, "metrics/particle/momentum/class_1/accepted/samples": 15983.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08745691180229187, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3211987316608429, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.396609365940094, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3352678716182709, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04906078800559044, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.85306167602539, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006942013744264841, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3426400125026703, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.011535964906215668, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.591303586959839, "metrics/particle/momentum/class_1/assigned/samples": 17142.0, "metrics/particle/momentum/class_1/efficiency": 0.9323338866233826, "metrics/particle/momentum/class_1/event/energy_bias": 2.1902999877929688, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.648576736450195, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.1778459548950195, "metrics/particle/momentum/class_1/event/px_bias": 0.5347196459770203, "metrics/particle/momentum/class_1/event/py_bias": 0.1702064871788025, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.64461898803711, "metrics/particle/momentum/class_1/event/pz_bias": -0.7086032629013062, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.509294509887695, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04000067338347435, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19619473814964294, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1986464560031891, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03453516215085983, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2375994771718979, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.034090910106897354, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6577174663543701, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.33170485496520996, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7649953365325928, "metrics/particle/momentum/class_2/accepted/samples": 1232.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8980793952941895, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7177878618240356, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 36530.0234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 36529.56640625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37867504358291626, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1844985.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.076832614839077, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0533125400543213, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3328758478164673, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.909452438354492, "metrics/particle/momentum/class_2/assigned/samples": 2551.0, "metrics/particle/momentum/class_2/efficiency": 0.4262975752353668, "metrics/particle/momentum/class_2/event/energy_bias": -9.541779518127441, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.761957168579102, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.462804794311523, "metrics/particle/momentum/class_2/event/px_bias": 0.0819740742444992, "metrics/particle/momentum/class_2/event/py_bias": 0.07729160040616989, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.866973876953125, "metrics/particle/momentum/class_2/event/pz_bias": -1.333634853363037, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.074760437011719, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029778581112623215, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13870051503181458, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13708193600177765, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.026790309697389603, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1781674176454544, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008554320083931088, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19220083951950073, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05425643548369408, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6417253613471985, "metrics/particle/momentum/class_3/accepted/samples": 11690.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30483588576316833, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4599562883377075, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5959007740020752, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4634038209915161, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15400944650173187, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.740676879882812, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008149010129272938, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4877146780490875, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03669895976781845, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1908445358276367, "metrics/particle/momentum/class_3/assigned/samples": 14603.0, "metrics/particle/momentum/class_3/efficiency": 0.7696359157562256, "metrics/particle/momentum/class_3/event/energy_bias": 6.189474105834961, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.768860816955566, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.755611419677734, "metrics/particle/momentum/class_3/event/px_bias": 0.009812355041503906, "metrics/particle/momentum/class_3/event/py_bias": -0.5354999899864197, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.648269176483154, "metrics/particle/momentum/class_3/event/pz_bias": -0.9837005138397217, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.039868354797363, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013568929396569729, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10590367764234543, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10647553950548172, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0455959290266037, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15749245882034302, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006688963156193495, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4886397421360016, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07369458675384521, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6957796812057495, "metrics/particle/momentum/class_4/accepted/samples": 299.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22112533450126648, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.062452793121338, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.053990364074707, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.9855759739875793, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12711864709854126, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.803533554077148, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6306027173995972, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.16338206827640533, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.3220551013946533, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8446327447891235, "metrics/particle/momentum/class_4/event/energy_bias": -1.7313846349716187, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.097231864929199, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5952578783035278, "metrics/particle/momentum/class_4/event/px_bias": -0.34305524826049805, "metrics/particle/momentum/class_4/event/py_bias": 0.13726860284805298, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0810275077819824, "metrics/particle/momentum/class_4/event/pz_bias": 0.5595487356185913, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8487744331359863, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007229120470583439, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026408765465021133, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025258837267756462, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009564410895109177, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07018066942691803, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15336564183235168, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.009598757140338421, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.45642900466918945, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013143317773938179, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15618427097797394, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1563805490732193, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11316131055355072, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.3794589042663574, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.23529987037181854, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08618201315402985, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8407994508743286, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": 0.1911763995885849, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.030686378479004, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7706016898155212, "metrics/particle/momentum/class_5/event/px_bias": 0.18117623031139374, "metrics/particle/momentum/class_5/event/py_bias": 0.0008390792645514011, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5371347069740295, "metrics/particle/momentum/class_5/event/pz_bias": 0.16721270978450775, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.9361484050750732, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3305022716522217, "step_valid_time": 4915.039520263672, "step_total_time": 4916.370023012161, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004928730512892, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08824997272404789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2917, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2901, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9573350836888743, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9598552155314248, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9585934932632271, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04014478446857519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520840170659666, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9545903257650543, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9533355241537956, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9573350836888743, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001981703720688, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09596573027957689, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2146, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2136, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9420544337137841, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9432967032967033, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9426751592356688, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056703296703296706, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9376646180860404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9389010989010989, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9382824511311223, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9420544337137841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005916063854167, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09506640496401442, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1308, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1214, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1211, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.921092564491654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9281345565749235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9246001523229246, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07186544342507645, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9188163884673748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9258409785932722, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9223153084539223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.921092564491654} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_29000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..2024cccbe06a40af2ab2a5182269443a775459e2 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3736263736263736, "train": {"Classification_binary": 2.7443525791168213, "Classification": 0.02045503817498684, "Regression_pt": 0.15551936626434326, "Regression_eta": 0.009779141284525394, "Regression_sin_phi": 0.006902893539518118, "Regression_cos_phi": 0.007108147256076336, "Regression_energy": 0.15837182104587555, "Total": 8.644407272338867, "Total_unweighted": 3.102488986682147}, "train_group": {}, "valid": {"Classification_binary": 2.71617062886556, "Classification": 0.019168848792711895, "Regression_pt": 0.14553438623746237, "Regression_eta": 0.012008981158336004, "Regression_sin_phi": 0.0069716498255729675, "Regression_cos_phi": 0.0074036139994859695, "Regression_energy": 0.1511989633242289, "Total": 8.49587631225586, "Total_unweighted": 3.058457072203358, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01906745694577694, "diagnostic/loss/cld_pf/Classification_binary": 2.7099575996398926, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007039892487227917, "diagnostic/loss/cld_pf/Regression_energy": 0.14989593625068665, "diagnostic/loss/cld_pf/Regression_eta": 0.011122320778667927, "diagnostic/loss/cld_pf/Regression_pt": 0.14526095986366272, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006530199199914932, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07992832362651825, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20436730980873108, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05280904099345207, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4355384409427643, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.042670801281929016, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19915315508842468, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06983793526887894, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4309368431568146, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9288589358329773, "metrics/particle/class_1/pid_accuracy": 0.9851029515266418, "metrics/particle/class_2/efficiency": 0.43386122584342957, "metrics/particle/class_2/pid_accuracy": 0.7486869692802429, "metrics/particle/class_3/efficiency": 0.7704377770423889, "metrics/particle/class_3/pid_accuracy": 0.9815448522567749, "metrics/particle/class_4/efficiency": 0.8224016427993774, "metrics/particle/class_4/pid_accuracy": 0.3042944669723511, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6101009845733643, "metrics/particle/count/bias_mean": -0.47647058963775635, "metrics/particle/count/mae": 3.3614377975463867, "metrics/particle/count/prediction_mean": 70.0888900756836, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05524686351418495, "metrics/particle/event/energy_response_mean": 0.9875820875167847, "metrics/particle/event/met_abs_error": 5.652478218078613, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04393273591995239, "metrics/particle/event/scalar_pt_response_mean": 0.9974327683448792, "metrics/particle/event/vector_pt_closure": 0.05054078623652458, "metrics/particle/matched/delta_eta_abs_mean": 0.00974553544074297, "metrics/particle/matched/delta_phi_abs_mean": 0.010355846956372261, "metrics/particle/matched/delta_r_mean": 0.015936756506562233, "metrics/particle/matched/energy_relative_abs_mean": 0.09355981647968292, "metrics/particle/matched/pid_accuracy": 0.965185821056366, "metrics/particle/matched/pt_relative_abs_mean": 0.08686897903680801, "metrics/particle/matching/duplicate_fraction": 0.03398112580180168, "metrics/particle/matching/efficiency": 0.8210253119468689, "metrics/particle/matching/f1": 0.8238065838813782, "metrics/particle/matching/purity": 0.8266067504882812, "metrics/particle/matching/target_coverage_dr0p05": 0.8008058071136475, "metrics/particle/matching/target_coverage_dr0p10": 0.8583614826202393, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7122864127159119, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8581855297088623, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018240243196487427, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08883682638406754, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08538919687271118, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011106330901384354, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1376129686832428, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0057767946273088455, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.39656776189804077, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.011012035422027111, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6048802137374878, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8136.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27321794629096985, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9372578263282776, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9731798768043518, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8549792766571045, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1254575103521347, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.663352966308594, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014741765335202217, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7581138014793396, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.004250607453286648, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.704975128173828, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9836.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8026043176651001, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7730081081390381, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.179932594299316, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2080273628234863, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.3479856252670288, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.06984558701515198, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.1346540451049805, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.15003535151481628, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.962201118469238, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029664315283298492, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0983632504940033, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0997033342719078, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010522267781198025, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15339677035808563, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018366053700447083, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9760057330131531, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.34622859954833984, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.37216854095459, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1579.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7846308350563049, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.595780372619629, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 30594.2265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 30594.04296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27247849106788635, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1547202.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0828772485256195, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0612826347351074, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1894381046295166, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.156780242919922, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2558.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.574390709400177, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.5186498165130615, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.735151290893555, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1992859840393066, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.046564385294914246, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07141561806201935, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.575420379638672, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6539971828460693, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.088903427124023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013916785828769207, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09670888632535934, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08855967968702316, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01590033993124962, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14407393336296082, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000509735953528434, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11725318431854248, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0018969924421980977, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.512584388256073, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19618.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15764905512332916, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7343310117721558, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5902076363563538, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4738924205303192, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09753268212080002, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.299491882324219, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034355062525719404, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27492520213127136, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007388715632259846, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5601879358291626, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22413.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8580676317214966, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.5410339832305908, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.365020275115967, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.211491584777832, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5568554401397705, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.8555418848991394, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.293456792831421, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2775819003582001, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.988027095794678, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015963660553097725, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09461449086666107, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08828014135360718, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014281138777732849, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1428348273038864, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002931851428002119, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24095255136489868, "metrics/particle/momentum/all/accepted/pz_bias": -0.02296062745153904, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.225663423538208, "metrics/particle/momentum/all/accepted/samples": 29333.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2363847941160202, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8549842238426208, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2249.053955078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2248.9326171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1182808056473732, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 419434.46875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012468756176531315, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6162400245666504, "metrics/particle/momentum/all/assigned/pz_bias": -0.010365397669374943, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1300244331359863, "metrics/particle/momentum/all/assigned/samples": 34807.0, "metrics/particle/momentum/all/efficiency": 0.8205264210700989, "metrics/particle/momentum/all/event/energy_bias": -3.2866756916046143, "metrics/particle/momentum/all/event/p3_error_mean": 15.250505447387695, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.052228927612305, "metrics/particle/momentum/all/event/px_bias": -0.95140540599823, "metrics/particle/momentum/all/event/py_bias": 0.857111930847168, "metrics/particle/momentum/all/event/pz_abs_mean": 10.96345329284668, "metrics/particle/momentum/all/event/pz_bias": -0.22637997567653656, "metrics/particle/momentum/all/event/pz_residual_rms": 19.263137817382812, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008771299384534359, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05439222231507301, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04335980862379074, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009833346121013165, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09506205469369888, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017000377411022782, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1494286209344864, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.010577910579741001, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.880617082118988, "metrics/particle/momentum/class_1/accepted/samples": 15882.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09427621215581894, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2750491499900818, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3301326632499695, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2716462016105652, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05269915238022804, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.995869159698486, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005894368048757315, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3485911190509796, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0052879792638123035, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8047749996185303, "metrics/particle/momentum/class_1/assigned/samples": 17135.0, "metrics/particle/momentum/class_1/efficiency": 0.9264422655105591, "metrics/particle/momentum/class_1/event/energy_bias": 2.7424726486206055, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.170950889587402, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.106446266174316, "metrics/particle/momentum/class_1/event/px_bias": -0.3094358742237091, "metrics/particle/momentum/class_1/event/py_bias": 0.37048307061195374, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.125139236450195, "metrics/particle/momentum/class_1/event/pz_bias": 0.08244985342025757, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.874357223510742, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03670342639088631, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2005099207162857, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2043810486793518, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.054841600358486176, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24308627843856812, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04028436169028282, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8178170919418335, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.386801540851593, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.353221416473389, "metrics/particle/momentum/class_2/accepted/samples": 1266.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9141069054603577, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.781524121761322, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 30296.12109375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 30295.63671875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36546650528907776, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1539696.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08633372187614441, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1362595558166504, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13694430887699127, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.890239238739014, "metrics/particle/momentum/class_2/assigned/samples": 2583.0, "metrics/particle/momentum/class_2/efficiency": 0.43806228041648865, "metrics/particle/momentum/class_2/event/energy_bias": -9.219688415527344, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.155176162719727, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.239325523376465, "metrics/particle/momentum/class_2/event/px_bias": -0.5161385536193848, "metrics/particle/momentum/class_2/event/py_bias": 0.507689893245697, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.35632038116455, "metrics/particle/momentum/class_2/event/pz_bias": -1.1939753293991089, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.968106269836426, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02367168292403221, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13811343908309937, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13676950335502625, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.026771899312734604, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17777033150196075, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00042618479346856475, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1884823888540268, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004875273443758488, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7511687874794006, "metrics/particle/momentum/class_3/accepted/samples": 11732.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2853328585624695, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5299320220947266, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4802918434143066, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3464430570602417, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1514797806739807, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.01523208618164, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007278719916939735, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4874078035354614, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0036691713612526655, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3725619316101074, "metrics/particle/momentum/class_3/assigned/samples": 14563.0, "metrics/particle/momentum/class_3/efficiency": 0.7724010944366455, "metrics/particle/momentum/class_3/event/energy_bias": 4.962747573852539, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.22729206085205, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.324385643005371, "metrics/particle/momentum/class_3/event/px_bias": -0.1680816262960434, "metrics/particle/momentum/class_3/event/py_bias": -0.13882769644260406, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.3510050773620605, "metrics/particle/momentum/class_3/event/pz_bias": 0.544808566570282, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.017057418823242, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012099148705601692, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11563120037317276, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11738503724336624, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.059027332812547684, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17486537992954254, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010526316240429878, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5597819685935974, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13640369474887848, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9675711393356323, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2587277293205261, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.0939226150512695, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.056450128555298, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9866151809692383, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16713881492614746, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.017377853393555, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008498583920300007, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6787859797477722, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10487774014472961, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.026167631149292, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.805084764957428, "metrics/particle/momentum/class_4/event/energy_bias": -1.6213395595550537, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3023853302001953, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.719834804534912, "metrics/particle/momentum/class_4/event/px_bias": -0.1008293405175209, "metrics/particle/momentum/class_4/event/py_bias": 0.16991007328033447, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.257174015045166, "metrics/particle/momentum/class_4/event/pz_bias": 0.20981243252754211, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.674004077911377, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007888699881732464, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025728557258844376, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024406662210822105, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007465359754860401, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.061783622950315475, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1337299346923828, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.014920850284397602, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.45453765988349915, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02688653953373432, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.047607745975255966, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04637203365564346, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0019809752702713013, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.148877814412117, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21773585677146912, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.03992918133735657, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9466913342475891, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": -0.15086832642555237, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7894597053527832, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6567628979682922, "metrics/particle/momentum/class_5/event/px_bias": 0.1430799961090088, "metrics/particle/momentum/class_5/event/py_bias": -0.052143387496471405, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.34444770216941833, "metrics/particle/momentum/class_5/event/pz_bias": 0.13052457571029663, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.452784776687622, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3473825454711914, "step_valid_time": 5055.420375347137, "step_total_time": 5056.767758131027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989725041030636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.084361706150192, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2906, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9586478503446012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959592641261498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591200131341323, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04040735873850197, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9537249753856252, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.954664914586071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9541947135117387, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9586478503446012, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9968614621373393, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09261798665070065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2276, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2164, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2156, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9499561018437226, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507908611599297, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9503732981993852, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0492091388400703, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9464442493415277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9472759226713533, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9468599033816425, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9499561018437226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000385835027002, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.094266518027869, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1308, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1219, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.924886191198786, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9319571865443425, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9284082254379284, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06804281345565749, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9233687405159332, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9304281345565749, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9268849961919269, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.924886191198786} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_3000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..32c5e256af1c3ba49890936b0f011ce42cb6c246 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14209928003031452, "train": {"Classification_binary": 4.1296305656433105, "Classification": 0.032510656863451004, "Regression_pt": 0.25647756457328796, "Regression_eta": 0.013928143307566643, "Regression_sin_phi": 0.010400780476629734, "Regression_cos_phi": 0.012844514101743698, "Regression_energy": 0.2575297951698303, "Total": 13.655508041381836, "Total_unweighted": 4.71332202013582}, "train_group": {}, "valid": {"Classification_binary": 4.193251609802246, "Classification": 0.03265348821878433, "Regression_pt": 0.2514445384343465, "Regression_eta": 0.016006022691726685, "Regression_sin_phi": 0.009712656339009603, "Regression_cos_phi": 0.010053738330801329, "Regression_energy": 0.25655444463094074, "Total": 13.500478108723959, "Total_unweighted": 4.769676498447855, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.03239334002137184, "diagnostic/loss/cld_pf/Classification_binary": 4.179429054260254, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.009820293635129929, "diagnostic/loss/cld_pf/Regression_energy": 0.25982731580734253, "diagnostic/loss/cld_pf/Regression_eta": 0.015528256073594093, "diagnostic/loss/cld_pf/Regression_pt": 0.2545766234397888, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.009160585701465607, "diagnostic/regression/cld_pf/energy_pred_mean": 0.11105155199766159, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.26793017983436584, "diagnostic/regression/cld_pf/energy_residual_mean": 0.08393226563930511, "diagnostic/regression/cld_pf/energy_residual_rms": 0.543653666973114, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.0758918970823288, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.26613849401474, "diagnostic/regression/cld_pf/pt_residual_mean": 0.10305903106927872, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5420137047767639, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.8955881595611572, "metrics/particle/class_1/pid_accuracy": 0.9790926575660706, "metrics/particle/class_2/efficiency": 0.20861342549324036, "metrics/particle/class_2/pid_accuracy": 0.22064445912837982, "metrics/particle/class_3/efficiency": 0.6701169013977051, "metrics/particle/class_3/pid_accuracy": 0.9777653813362122, "metrics/particle/class_4/efficiency": 0.8829464912414551, "metrics/particle/class_4/pid_accuracy": 0.09600000083446503, "metrics/particle/class_5/efficiency": 0.9456310868263245, "metrics/particle/class_5/pid_accuracy": 0.2874743342399597, "metrics/particle/count/bias_mean": 1.074509859085083, "metrics/particle/count/mae": 4.963398456573486, "metrics/particle/count/prediction_mean": 71.6398696899414, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.10691036283969879, "metrics/particle/event/energy_response_mean": 0.9945060610771179, "metrics/particle/event/met_abs_error": 12.869314193725586, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09362941235303879, "metrics/particle/event/scalar_pt_response_mean": 0.9917434453964233, "metrics/particle/event/vector_pt_closure": 0.10383336991071701, "metrics/particle/matched/delta_eta_abs_mean": 0.017084401100873947, "metrics/particle/matched/delta_phi_abs_mean": 0.013271908275783062, "metrics/particle/matched/delta_r_mean": 0.024378616362810135, "metrics/particle/matched/energy_relative_abs_mean": 0.1093350350856781, "metrics/particle/matched/pid_accuracy": 0.9475228786468506, "metrics/particle/matched/pt_relative_abs_mean": 0.1045246571302414, "metrics/particle/matching/duplicate_fraction": 0.05081699416041374, "metrics/particle/matching/efficiency": 0.7448339462280273, "metrics/particle/matching/f1": 0.7392059564590454, "metrics/particle/matching/purity": 0.7336623668670654, "metrics/particle/matching/target_coverage_dr0p05": 0.707034707069397, "metrics/particle/matching/target_coverage_dr0p10": 0.7943963408470154, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6250914931297302, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7934979200363159, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027112416923046112, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11072985082864761, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.10865532606840134, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0363595150411129, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16329196095466614, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005412156693637371, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4953232407569885, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0037866116035729647, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8094961643218994, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7206.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4570842981338501, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.797563910484314, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7948637008666992, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6515613794326782, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20305810868740082, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.61333179473877, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.024362895637750626, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3713124990463257, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.22923575341701508, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 7.334052562713623, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9810.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7108612060546875, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.023136615753173828, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 14.003286361694336, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.431183815002441, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.09350071102380753, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6991521120071411, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.977859497070312, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -4.244710922241211, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.490070343017578, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03662877157330513, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13142113387584686, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12665767967700958, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.040558625012636185, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1811339110136032, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017001545056700706, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0371977090835571, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0497930608689785, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.048536539077759, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1294.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.059722900390625, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.023709297180176, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5832.712890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5832.51513671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.37734344601631165, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 291812.71875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.10291184484958649, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.120911121368408, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4683822691440582, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.605944633483887, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2507.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4707166254520416, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 4.4344401359558105, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.837968826293945, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.9619983434677124, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04021234065294266, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.17987284064292908, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.57762336730957, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.165864109992981, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.37881088256836, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02249993570148945, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10878781974315643, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1033654659986496, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024762731045484543, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15652361512184143, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003288932784926146, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14605054259300232, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02587379328906536, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5129237771034241, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18243.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.25740164518356323, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0472986698150635, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8447179794311523, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.685603678226471, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1520501673221588, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.389900207519531, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007337898947298527, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.49181556701660156, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0017073435010388494, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.5344021320343018, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22486.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7979267835617065, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -5.774184703826904, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.922567367553711, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.457050323486328, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.9233462810516357, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.9872140884399414, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.160238265991211, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8569859862327576, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.790571212768555, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024426432326436043, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11040625721216202, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10591786354780197, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.028651833534240723, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15964019298553467, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002505328506231308, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2832828462123871, "metrics/particle/momentum/all/accepted/pz_bias": 0.019039057195186615, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2293978929519653, "metrics/particle/momentum/all/accepted/samples": 26743.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3714810907840729, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3291125297546387, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 421.2054443359375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 421.0479736328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1826566606760025, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 78320.0703125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.019021349027752876, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.001139760017395, "metrics/particle/momentum/all/assigned/pz_bias": -0.09725155681371689, "metrics/particle/momentum/all/assigned/pz_residual_rms": 5.386488437652588, "metrics/particle/momentum/all/assigned/samples": 34803.0, "metrics/particle/momentum/all/efficiency": 0.748076856136322, "metrics/particle/momentum/all/event/energy_bias": -1.316608190536499, "metrics/particle/momentum/all/event/p3_error_mean": 31.844776153564453, "metrics/particle/momentum/all/event/pt_vector_error_mean": 16.972671508789062, "metrics/particle/momentum/all/event/px_bias": -1.9766347408294678, "metrics/particle/momentum/all/event/py_bias": 3.866239070892334, "metrics/particle/momentum/all/event/pz_abs_mean": 23.078527450561523, "metrics/particle/momentum/all/event/pz_bias": -6.267560958862305, "metrics/particle/momentum/all/event/pz_residual_rms": 35.04972839355469, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019738350063562393, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06743767112493515, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0606229342520237, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008935090154409409, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10732316225767136, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012385111767798662, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.20162543654441833, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06990315020084381, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9434230923652649, "metrics/particle/momentum/class_1/accepted/samples": 15341.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.147945374250412, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.39812734723091125, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5134394764900208, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.42308223247528076, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07849889248609543, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.935022830963135, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009104704484343529, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5359054207801819, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.013433772139251232, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.5311050415039062, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.8948842287063599, "metrics/particle/momentum/class_1/event/energy_bias": 3.8462491035461426, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.74785804748535, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.15891170501709, "metrics/particle/momentum/class_1/event/px_bias": -0.9877769947052002, "metrics/particle/momentum/class_1/event/py_bias": 0.46450191736221313, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.528114318847656, "metrics/particle/momentum/class_1/event/pz_bias": 0.01347583532333374, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.32469940185547, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.045948248356580734, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23485170304775238, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22943152487277985, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08293410390615463, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27201953530311584, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.057877812534570694, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.2698869705200195, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.4036160707473755, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.226068496704102, "metrics/particle/momentum/class_2/accepted/samples": 622.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.4453563690185547, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.141860008239746, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5860.05029296875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5859.31005859375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5765838027000427, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 292572.28125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.1303127557039261, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 5.037135124206543, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.6791487336158752, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 12.182680130004883, "metrics/particle/momentum/class_2/assigned/samples": 2494.0, "metrics/particle/momentum/class_2/efficiency": 0.21522490680217743, "metrics/particle/momentum/class_2/event/energy_bias": -19.70767593383789, "metrics/particle/momentum/class_2/event/p3_error_mean": 25.121742248535156, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.458386421203613, "metrics/particle/momentum/class_2/event/px_bias": -0.16240042448043823, "metrics/particle/momentum/class_2/event/py_bias": 1.051445722579956, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.74854850769043, "metrics/particle/momentum/class_2/event/pz_bias": -2.4373440742492676, "metrics/particle/momentum/class_2/event/pz_residual_rms": 33.51163101196289, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030250733718276024, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16821840405464172, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16707207262516022, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.06617724150419235, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2091335654258728, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008728542597964406, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2723803222179413, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.038766488432884216, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8475360870361328, "metrics/particle/momentum/class_3/accepted/samples": 10311.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.45888686180114746, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4714038372039795, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3927226066589355, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2541871070861816, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24044238030910492, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.204959869384766, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012015292420983315, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.869335412979126, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.1089668720960617, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.370510578155518, "metrics/particle/momentum/class_3/assigned/samples": 14648.0, "metrics/particle/momentum/class_3/efficiency": 0.678846538066864, "metrics/particle/momentum/class_3/event/energy_bias": 18.7530517578125, "metrics/particle/momentum/class_3/event/p3_error_mean": 17.48206901550293, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.37252426147461, "metrics/particle/momentum/class_3/event/px_bias": -1.082505226135254, "metrics/particle/momentum/class_3/event/py_bias": 2.4857687950134277, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.670360565185547, "metrics/particle/momentum/class_3/event/pz_bias": -3.4772186279296875, "metrics/particle/momentum/class_3/event/pz_residual_rms": 23.035940170288086, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023369889706373215, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10095154494047165, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10123087465763092, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0037790979258716106, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15230828523635864, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009836065582931042, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6281723380088806, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.21287143230438232, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6912550926208496, "metrics/particle/momentum/class_4/accepted/samples": 305.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1355293244123459, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.050421118736267, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.094621181488037, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.9862846732139587, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1214689239859581, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.3354692459106445, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.786892294883728, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.29697275161743164, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.0553951263427734, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8615819215774536, "metrics/particle/momentum/class_4/event/energy_bias": -3.658740520477295, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.7413625717163086, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.7626185417175293, "metrics/particle/momentum/class_4/event/px_bias": 0.16120034456253052, "metrics/particle/momentum/class_4/event/py_bias": 0.25039395689964294, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0215747356414795, "metrics/particle/momentum/class_4/event/pz_bias": 0.23670904338359833, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.768179893493652, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017116598784923553, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.04064105078577995, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.038312021642923355, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.01681934855878353, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06864818930625916, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.4312313497066498, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.13793812692165375, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9942011833190918, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.11149924248456955, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.08561523258686066, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.08333779126405716, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.024250367656350136, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05202312022447586, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.22660322487354279, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4928516149520874, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.17539669573307037, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0824902057647705, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": -0.5494911074638367, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.733546495437622, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.009439468383789, "metrics/particle/momentum/class_5/event/px_bias": 0.09484746307134628, "metrics/particle/momentum/class_5/event/py_bias": -0.3858714699745178, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5177226066589355, "metrics/particle/momentum/class_5/event/pz_bias": -0.603183388710022, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.911656856536865, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.1666224002838135, "step_valid_time": 677.4246077537537, "step_total_time": 678.591230392456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.980497054208431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20412976351057344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2738, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8985887758450936, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9021416803953871, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9003617231173956, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09785831960461285, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8647850344601247, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8682042833607908, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8664912857612628, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8985887758450936, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9914039043928465, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21894468902757613, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2270, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2019, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1950, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8863037752414399, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8894273127753304, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8878627968337731, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1105726872246696, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8560140474100087, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8590308370044053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8575197889182058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8863037752414399, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9953271958464218, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20551186727871706, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1300, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1064, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8330804248861912, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8446153846153847, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8388082505729565, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15538461538461537, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8072837632776935, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8184615384615385, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8128342245989305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8330804248861912} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_30000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..fd2438117ef9344b7a7d30816b839ea0aa14a84f --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4209928003031451, "train": {"Classification_binary": 2.7382638454437256, "Classification": 0.019184114411473274, "Regression_pt": 0.14820615947246552, "Regression_eta": 0.013535135425627232, "Regression_sin_phi": 0.009850549511611462, "Regression_cos_phi": 0.007883820682764053, "Regression_energy": 0.15780241787433624, "Total": 8.994185447692871, "Total_unweighted": 3.0947260428220034}, "train_group": {}, "valid": {"Classification_binary": 2.7650070190429688, "Classification": 0.01931494226058324, "Regression_pt": 0.14755629499753317, "Regression_eta": 0.01239423950513204, "Regression_sin_phi": 0.006960528592268626, "Regression_cos_phi": 0.007540243367354075, "Regression_energy": 0.15373210112253824, "Total": 8.626928329467773, "Total_unweighted": 3.112505368888378, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019334960728883743, "diagnostic/loss/cld_pf/Classification_binary": 2.7637312412261963, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007142162416130304, "diagnostic/loss/cld_pf/Regression_energy": 0.15313033759593964, "diagnostic/loss/cld_pf/Regression_eta": 0.011185484007000923, "diagnostic/loss/cld_pf/Regression_pt": 0.147967129945755, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006552905775606632, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07540500909090042, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20530325174331665, "diagnostic/regression/cld_pf/energy_residual_mean": 0.048285722732543945, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4424367845058441, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.038493331521749496, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1995704621076584, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06566046178340912, "diagnostic/regression/cld_pf/pt_residual_rms": 0.43557026982307434, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9314845204353333, "metrics/particle/class_1/pid_accuracy": 0.985165536403656, "metrics/particle/class_2/efficiency": 0.4351145029067993, "metrics/particle/class_2/pid_accuracy": 0.741555392742157, "metrics/particle/class_3/efficiency": 0.766811728477478, "metrics/particle/class_3/pid_accuracy": 0.9832344651222229, "metrics/particle/class_4/efficiency": 0.8446013927459717, "metrics/particle/class_4/pid_accuracy": 0.2879330813884735, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.5846773982048035, "metrics/particle/count/bias_mean": -0.5732026100158691, "metrics/particle/count/mae": 3.5640523433685303, "metrics/particle/count/prediction_mean": 69.99215698242188, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05542922765016556, "metrics/particle/event/energy_response_mean": 0.9878102540969849, "metrics/particle/event/met_abs_error": 5.7049360275268555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04594261944293976, "metrics/particle/event/scalar_pt_response_mean": 0.9979392290115356, "metrics/particle/event/vector_pt_closure": 0.05046769604086876, "metrics/particle/matched/delta_eta_abs_mean": 0.00930106546729803, "metrics/particle/matched/delta_phi_abs_mean": 0.01040733139961958, "metrics/particle/matched/delta_r_mean": 0.01564745418727398, "metrics/particle/matched/energy_relative_abs_mean": 0.09346958249807358, "metrics/particle/matched/pid_accuracy": 0.9650866389274597, "metrics/particle/matched/pt_relative_abs_mean": 0.08721105009317398, "metrics/particle/matching/duplicate_fraction": 0.034093454480171204, "metrics/particle/matching/efficiency": 0.8210809230804443, "metrics/particle/matching/f1": 0.8244293332099915, "metrics/particle/matching/purity": 0.8278051614761353, "metrics/particle/matching/target_coverage_dr0p05": 0.8015004992485046, "metrics/particle/matching/target_coverage_dr0p10": 0.855860710144043, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7101467847824097, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8575464487075806, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017817657440900803, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08819422125816345, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08567464351654053, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010613271035254002, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1383432000875473, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004708214662969112, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38657647371292114, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.016990309581160545, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5489648580551147, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8071.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2688210606575012, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0094763040542603, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9833335280418396, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8636176586151123, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1268916130065918, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.865018367767334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015439672395586967, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.796055257320404, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.03262217342853546, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.105008602142334, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9780.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7961921691894531, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.0673149973154068, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.729999542236328, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4465508460998535, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1599734127521515, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2915290594100952, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.674866676330566, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3589884638786316, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.049734115600586, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027184806764125824, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1036914736032486, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10398723185062408, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012126627378165722, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15837177634239197, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026417525485157967, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1829638481140137, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.4104599952697754, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.211089134216309, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1552.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7700241804122925, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7835687398910522, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12319.3017578125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12319.11328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2720472514629364, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 620691.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07992126047611237, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.874232530593872, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3359111249446869, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.9735589027404785, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2540.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5645689368247986, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.2361469268798828, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.857513427734375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1824734210968018, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.16295892000198364, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09879358112812042, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.7112250328063965, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.8288066387176514, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.390106201171875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013620018027722836, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09547565132379532, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08772572875022888, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011010674759745598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14288783073425293, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004564588889479637, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11441236734390259, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005581857170909643, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.497124582529068, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19717.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15518134832382202, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6144287586212158, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.475394070148468, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3554973304271698, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09407463669776917, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.520680904388428, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003388470271602273, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.26476576924324036, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.010448909364640713, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2251111268997192, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22429.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8623977899551392, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.4987865686416626, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.552692413330078, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.276274681091309, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4807414412498474, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.16090184450149536, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5164639949798584, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.032622531056404114, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.569750785827637, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015492265112698078, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09390723705291748, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08802168816328049, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010960386134684086, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14252051711082458, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029993182979524136, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24580393731594086, "metrics/particle/momentum/all/accepted/pz_bias": -0.022634804248809814, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3282039165496826, "metrics/particle/momentum/all/accepted/samples": 29340.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2321072369813919, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8110727071762085, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 901.0706787109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 900.94580078125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11631989479064941, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 167811.3125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012374456971883774, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.605036199092865, "metrics/particle/momentum/all/assigned/pz_bias": -0.02699071727693081, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8860812187194824, "metrics/particle/momentum/all/assigned/samples": 34749.0, "metrics/particle/momentum/all/efficiency": 0.8207222819328308, "metrics/particle/momentum/all/event/energy_bias": -2.802248001098633, "metrics/particle/momentum/all/event/p3_error_mean": 15.68175983428955, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.075328826904297, "metrics/particle/momentum/all/event/px_bias": -0.15780913829803467, "metrics/particle/momentum/all/event/py_bias": -0.03183360397815704, "metrics/particle/momentum/all/event/pz_abs_mean": 11.476425170898438, "metrics/particle/momentum/all/event/pz_bias": -3.2204177379608154, "metrics/particle/momentum/all/event/pz_residual_rms": 18.755035400390625, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008737526834011078, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.053399864584207535, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04217357560992241, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008888470008969307, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09237558394670486, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016950216377153993, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1500968337059021, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0017892803298309445, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9774850010871887, "metrics/particle/momentum/class_1/accepted/samples": 15929.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09174180775880814, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2591758072376251, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.32647112011909485, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.266666442155838, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.051353875547647476, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.091450214385986, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0064192344434559345, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3343227505683899, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.002533972728997469, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.18280029296875, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9291839003562927, "metrics/particle/momentum/class_1/event/energy_bias": 1.3248459100723267, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.16879653930664, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.21846866607666, "metrics/particle/momentum/class_1/event/px_bias": 0.36755990982055664, "metrics/particle/momentum/class_1/event/py_bias": 0.3306915760040283, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.12369441986084, "metrics/particle/momentum/class_1/event/pz_bias": -0.5400939583778381, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.294596672058105, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035719241946935654, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19868670403957367, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20148108899593353, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.036476872861385345, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24254527688026428, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03755868598818779, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8015732765197754, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.27661263942718506, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.179734230041504, "metrics/particle/momentum/class_2/accepted/samples": 1278.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.852938175201416, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7343723177909851, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12168.1943359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12167.71875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3601711392402649, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 616937.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07545702159404755, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.990229845046997, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.14158089458942413, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.656642436981201, "metrics/particle/momentum/class_2/assigned/samples": 2571.0, "metrics/particle/momentum/class_2/efficiency": 0.4422145187854767, "metrics/particle/momentum/class_2/event/energy_bias": -7.287039756774902, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.717382431030273, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.56548547744751, "metrics/particle/momentum/class_2/event/px_bias": -0.5602638721466064, "metrics/particle/momentum/class_2/event/py_bias": 0.12692715227603912, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.730772018432617, "metrics/particle/momentum/class_2/event/pz_bias": -2.0577852725982666, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.051523208618164, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022707555443048477, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13837574422359467, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13859166204929352, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01814422197639942, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17957983911037445, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000856531027238816, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20117685198783875, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.029047705233097076, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0689774751663208, "metrics/particle/momentum/class_3/accepted/samples": 11675.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2902000844478607, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.485339879989624, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4445247650146484, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3028327226638794, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15025834739208221, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.572318077087402, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008405097760260105, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5049595832824707, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.042996007949113846, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4988577365875244, "metrics/particle/momentum/class_3/assigned/samples": 14515.0, "metrics/particle/momentum/class_3/efficiency": 0.7686483860015869, "metrics/particle/momentum/class_3/event/energy_bias": 4.952118396759033, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.42671012878418, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.366277694702148, "metrics/particle/momentum/class_3/event/px_bias": 0.14278122782707214, "metrics/particle/momentum/class_3/event/py_bias": -0.6776791214942932, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.646662712097168, "metrics/particle/momentum/class_3/event/pz_bias": -1.1117687225341797, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.739105224609375, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011867023073136806, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10727311670780182, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10789859294891357, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04335075989365578, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1612025797367096, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010309278033673763, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5098474025726318, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.18610642850399017, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7000524997711182, "metrics/particle/momentum/class_4/accepted/samples": 291.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.23143871128559113, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.7947354316711426, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.8589522242546082, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7853223085403442, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1384180784225464, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.053445339202881, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6782037615776062, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2779788076877594, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.9622397422790527, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8220338821411133, "metrics/particle/momentum/class_4/event/energy_bias": -1.6940197944641113, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.06048583984375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5582915544509888, "metrics/particle/momentum/class_4/event/px_bias": -0.1992049217224121, "metrics/particle/momentum/class_4/event/py_bias": 0.20131933689117432, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0832630395889282, "metrics/particle/momentum/class_4/event/pz_bias": 0.37979164719581604, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.7912285327911377, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006884769070893526, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023694034665822983, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02290111407637596, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012945336289703846, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06835048645734787, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12861663103103638, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.01725880429148674, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.42766979336738586, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.036524150520563126, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07867718487977982, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06810498982667923, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03124229609966278, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3684893846511841, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21962247788906097, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.02769528143107891, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8622995615005493, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.09815289080142975, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0451395511627197, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.798714280128479, "metrics/particle/momentum/class_5/event/px_bias": 0.0913185104727745, "metrics/particle/momentum/class_5/event/py_bias": -0.01309257186949253, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5185829997062683, "metrics/particle/momentum/class_5/event/pz_bias": 0.10943856835365295, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.8899073600769043, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3918840885162354, "step_valid_time": 5195.939910173416, "step_total_time": 5197.331794500351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020175920149685, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08671808741043985, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9621959237343852, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9614058137625225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03780407626561473, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9566787003610109, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9582511505588429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574642798489079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9606170003281916, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028889925824613, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09662488192160157, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2285, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2161, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9483588621444201, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498137190444883, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05164113785557987, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9486391571553995, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9457330415754923, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9471838702607933, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9512730465320457, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028687192780459, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10227140848163219, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1312, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1221, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9301972685887708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9344512195121951, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.932319391634981, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06554878048780488, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9264036418816388, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.930640243902439, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9285171102661597, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9301972685887708} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_31000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..e1006d1eb78525334a3618f0bcbad8f338dd8b37 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4683592269799166, "train": {"Classification_binary": 2.7166314125061035, "Classification": 0.02050366811454296, "Regression_pt": 0.15508000552654266, "Regression_eta": 0.01108382735401392, "Regression_sin_phi": 0.007324525620788336, "Regression_cos_phi": 0.007186705246567726, "Regression_energy": 0.15746130049228668, "Total": 8.70653247833252, "Total_unweighted": 3.075271444860846}, "train_group": {}, "valid": {"Classification_binary": 2.7284673055013022, "Classification": 0.019250482320785522, "Regression_pt": 0.14671042561531067, "Regression_eta": 0.011664662510156631, "Regression_sin_phi": 0.006940093512336413, "Regression_cos_phi": 0.007256743808587392, "Regression_energy": 0.15258095661799112, "Total": 8.512456258138021, "Total_unweighted": 3.07287066988647, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019400589168071747, "diagnostic/loss/cld_pf/Classification_binary": 2.7287001609802246, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006944657303392887, "diagnostic/loss/cld_pf/Regression_energy": 0.1518436074256897, "diagnostic/loss/cld_pf/Regression_eta": 0.011012906208634377, "diagnostic/loss/cld_pf/Regression_pt": 0.14663410186767578, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0064982203766703606, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08854604512453079, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20458565652370453, "diagnostic/regression/cld_pf/energy_residual_mean": 0.061426762491464615, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4341357946395874, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04956422746181488, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19790397584438324, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0767313688993454, "diagnostic/regression/cld_pf/pt_residual_rms": 0.42713403701782227, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9314270615577698, "metrics/particle/class_1/pid_accuracy": 0.9848765134811401, "metrics/particle/class_2/efficiency": 0.4253161549568176, "metrics/particle/class_2/pid_accuracy": 0.7299758791923523, "metrics/particle/class_3/efficiency": 0.7648338675498962, "metrics/particle/class_3/pid_accuracy": 0.9830761551856995, "metrics/particle/class_4/efficiency": 0.8314833641052246, "metrics/particle/class_4/pid_accuracy": 0.32281553745269775, "metrics/particle/class_5/efficiency": 0.9495145678520203, "metrics/particle/class_5/pid_accuracy": 0.6564416885375977, "metrics/particle/count/bias_mean": -0.929411768913269, "metrics/particle/count/mae": 3.4823529720306396, "metrics/particle/count/prediction_mean": 69.63594818115234, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05630724877119064, "metrics/particle/event/energy_response_mean": 0.9930224418640137, "metrics/particle/event/met_abs_error": 6.2017316818237305, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04717586189508438, "metrics/particle/event/scalar_pt_response_mean": 1.0035088062286377, "metrics/particle/event/vector_pt_closure": 0.05162323638796806, "metrics/particle/matched/delta_eta_abs_mean": 0.00946403481066227, "metrics/particle/matched/delta_phi_abs_mean": 0.009673572145402431, "metrics/particle/matched/delta_r_mean": 0.015191880986094475, "metrics/particle/matched/energy_relative_abs_mean": 0.09534788131713867, "metrics/particle/matched/pid_accuracy": 0.9654263854026794, "metrics/particle/matched/pt_relative_abs_mean": 0.08821683377027512, "metrics/particle/matching/duplicate_fraction": 0.033892419189214706, "metrics/particle/matching/efficiency": 0.8192377090454102, "metrics/particle/matching/f1": 0.8246685266494751, "metrics/particle/matching/purity": 0.8301718831062317, "metrics/particle/matching/target_coverage_dr0p05": 0.8066965937614441, "metrics/particle/matching/target_coverage_dr0p10": 0.8575371503829956, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7050618529319763, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8540267944335938, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017021434381604195, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09223489463329315, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08788701891899109, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.028909146785736084, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14143261313438416, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005309952888637781, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.39505335688591003, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.006804800592362881, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5088180303573608, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8098.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2701713442802429, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.935774028301239, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9361106157302856, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8276869654655457, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1273396760225296, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.45177936553955, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016364937648177147, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7932754158973694, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03342025727033615, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.957289934158325, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9777.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7988556623458862, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.1522165536880493, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.515754222869873, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3176705837249756, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.36808252334594727, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1475103497505188, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.503349304199219, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.20708303153514862, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.922416687011719, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02636527828872204, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10180414468050003, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.101195327937603, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.027010545134544373, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1540195196866989, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023899370804429054, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9981200695037842, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12203050404787064, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.414153814315796, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1590.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7734220623970032, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5427318811416626, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21465.013671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21464.849609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2710866928100586, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1083548.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08238524943590164, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0118794441223145, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1517343819141388, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.910359382629395, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2549.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5783921480178833, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.8340106010437012, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.065106391906738, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0603002309799194, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07345463335514069, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09064052999019623, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.9118852615356445, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3657571077346802, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.372373580932617, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013467429205775261, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09719967097043991, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08886618912220001, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024456284940242767, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14436744153499603, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004599815874826163, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11867053806781769, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00028653652407228947, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.49539580941200256, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19566.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15516828000545502, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7028152346611023, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5816991925239563, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47242456674575806, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09991494566202164, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.070231914520264, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033125923946499825, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2727368175983429, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02567804418504238, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3364322185516357, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22339.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8557931780815125, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.3298020362854004, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.473438262939453, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.271025657653809, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6155980229377747, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.9011892080307007, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.3899121284484863, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.865188717842102, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.190249919891357, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01515225600451231, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09607559442520142, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08926524966955185, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.025827739387750626, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1441051959991455, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030765023548156023, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24297739565372467, "metrics/particle/momentum/all/accepted/pz_bias": -0.00870787538588047, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1949355602264404, "metrics/particle/momentum/all/accepted/samples": 29254.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23306569457054138, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8302805423736572, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1579.0125732421875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1578.899169921875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12023655325174332, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 293824.28125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012808308005332947, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6209664940834045, "metrics/particle/momentum/all/assigned/pz_bias": 0.014816117472946644, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0750060081481934, "metrics/particle/momentum/all/assigned/samples": 34665.0, "metrics/particle/momentum/all/efficiency": 0.8183165788650513, "metrics/particle/momentum/all/event/energy_bias": -1.6564252376556396, "metrics/particle/momentum/all/event/p3_error_mean": 15.566549301147461, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.195789337158203, "metrics/particle/momentum/all/event/px_bias": 0.9102259278297424, "metrics/particle/momentum/all/event/py_bias": 0.8443194031715393, "metrics/particle/momentum/all/event/pz_abs_mean": 11.288949966430664, "metrics/particle/momentum/all/event/pz_bias": 1.4380288124084473, "metrics/particle/momentum/all/event/pz_residual_rms": 19.724727630615234, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008960208855569363, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.054632194340229034, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.043610282242298126, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011387011967599392, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09469275176525116, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015075376722961664, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15438520908355713, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.014418291859328747, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9164206385612488, "metrics/particle/momentum/class_1/accepted/samples": 15920.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09031306207180023, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2572944164276123, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.29112356901168823, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23347097635269165, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0527053065598011, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.6312692165374756, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006186890881508589, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3573096692562103, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.005161208100616932, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6188571453094482, "metrics/particle/momentum/class_1/assigned/samples": 17133.0, "metrics/particle/momentum/class_1/efficiency": 0.9286589026451111, "metrics/particle/momentum/class_1/event/energy_bias": 3.260709285736084, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.680318832397461, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.562013626098633, "metrics/particle/momentum/class_1/event/px_bias": 1.1947498321533203, "metrics/particle/momentum/class_1/event/py_bias": 0.35192519426345825, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.399368286132812, "metrics/particle/momentum/class_1/event/pz_bias": -0.31934744119644165, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.81010627746582, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03651714697480202, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19530871510505676, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19741183519363403, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03631284087896347, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2371256798505783, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04193548485636711, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7467883825302124, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.28054317831993103, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.176340103149414, "metrics/particle/momentum/class_2/accepted/samples": 1240.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9209595322608948, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7651041746139526, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 21496.494140625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 21496.017578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3815324306488037, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1084399.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08408644050359726, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.174616575241089, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13579270243644714, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.374177932739258, "metrics/particle/momentum/class_2/assigned/samples": 2545.0, "metrics/particle/momentum/class_2/efficiency": 0.4290657341480255, "metrics/particle/momentum/class_2/event/energy_bias": -9.802666664123535, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.065361022949219, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.2309651374816895, "metrics/particle/momentum/class_2/event/px_bias": -0.23206153512001038, "metrics/particle/momentum/class_2/event/py_bias": 0.7577448487281799, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.2919282913208, "metrics/particle/momentum/class_2/event/pz_bias": -0.33500686287879944, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.810051918029785, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02153116464614868, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14283710718154907, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14057330787181854, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05193872004747391, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18226535618305206, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009447736665606499, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19784092903137207, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02649933658540249, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7052903175354004, "metrics/particle/momentum/class_3/accepted/samples": 11643.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2827479839324951, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4729455709457397, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5252494812011719, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4078646898269653, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1541493833065033, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.800503730773926, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008022130466997623, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.48519399762153625, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06315550953149796, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1964831352233887, "metrics/particle/momentum/class_3/assigned/samples": 14460.0, "metrics/particle/momentum/class_3/efficiency": 0.766541600227356, "metrics/particle/momentum/class_3/event/energy_bias": 6.330767631530762, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.657392501831055, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.612366199493408, "metrics/particle/momentum/class_3/event/px_bias": 0.2080419361591339, "metrics/particle/momentum/class_3/event/py_bias": -0.3318481743335724, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.59814977645874, "metrics/particle/momentum/class_3/event/pz_bias": 1.593872308731079, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.470447540283203, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012335543520748615, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11079413443803787, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11133173853158951, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.049400877207517624, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16277004778385162, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010489510372281075, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5370867252349854, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06809275597333908, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.810513973236084, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2525034248828888, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.15470552444458, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3767459392547607, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.320474624633789, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.657390594482422, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01977401040494442, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7438328862190247, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1392521858215332, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.343763828277588, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.5250643491744995, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1260054111480713, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.66795015335083, "metrics/particle/momentum/class_4/event/px_bias": -0.36361193656921387, "metrics/particle/momentum/class_4/event/py_bias": 0.01715869829058647, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0388184785842896, "metrics/particle/momentum/class_4/event/pz_bias": 0.42060065269470215, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.674656867980957, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0067938026040792465, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02380896545946598, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022808030247688293, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.002789015183225274, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06317369639873505, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16461430490016937, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0323256216943264, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5158674716949463, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05851832777261734, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0618627704679966, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0568620041012764, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0015898378333076835, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.040462426841259, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.18716035783290863, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.262401282787323, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08614927530288696, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.004787564277649, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": 0.07982933521270752, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9155911803245544, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7496350407600403, "metrics/particle/momentum/class_5/event/px_bias": 0.10310768336057663, "metrics/particle/momentum/class_5/event/py_bias": 0.04933883622288704, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4241141676902771, "metrics/particle/momentum/class_5/event/pz_bias": 0.07791025936603546, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.66228449344635, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.392953634262085, "step_valid_time": 5336.460462093353, "step_total_time": 5337.853415966034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004635822630429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08922524914496499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038726616343944864, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9612733836560552, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043387454399875, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09327086845818522, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2164, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9559665345662703, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9544954935150582, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044033465433729636, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9499561018437226, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9528841919859093, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9514178940426468, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530289727831431, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005543291081186, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09924678100392748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1228, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9317147192716236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9295987887963664, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9306555513452065, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07040121120363362, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9294385432473444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.927327781983346, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9283819628647215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9317147192716236} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_32000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..23345d6a21909cda88b4753c2d0ff1c74e0960ca --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.515725653656688, "train": {"Classification_binary": 2.647758960723877, "Classification": 0.018294911831617355, "Regression_pt": 0.13541489839553833, "Regression_eta": 0.01989784464240074, "Regression_sin_phi": 0.006455599330365658, "Regression_cos_phi": 0.007829439826309681, "Regression_energy": 0.14046108722686768, "Total": 8.487557411193848, "Total_unweighted": 2.9761127419769764}, "train_group": {}, "valid": {"Classification_binary": 2.6798295974731445, "Classification": 0.018886490414539974, "Regression_pt": 0.142179141441981, "Regression_eta": 0.011738855391740799, "Regression_sin_phi": 0.006889264409740766, "Regression_cos_phi": 0.007283389568328857, "Regression_energy": 0.1478975017865499, "Total": 8.34529940287272, "Total_unweighted": 3.014704240486026, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018851079046726227, "diagnostic/loss/cld_pf/Classification_binary": 2.6818463802337646, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007073129527270794, "diagnostic/loss/cld_pf/Regression_energy": 0.14737415313720703, "diagnostic/loss/cld_pf/Regression_eta": 0.011081312783062458, "diagnostic/loss/cld_pf/Regression_pt": 0.14273592829704285, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006332961842417717, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08304189145565033, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20088578760623932, "diagnostic/regression/cld_pf/energy_residual_mean": 0.055922605097293854, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4296117126941681, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04883970320224762, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1964418590068817, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07600684463977814, "diagnostic/regression/cld_pf/pt_residual_rms": 0.42555415630340576, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.934895932674408, "metrics/particle/class_1/pid_accuracy": 0.9851171374320984, "metrics/particle/class_2/efficiency": 0.43921613693237305, "metrics/particle/class_2/pid_accuracy": 0.7260700464248657, "metrics/particle/class_3/efficiency": 0.7778876423835754, "metrics/particle/class_3/pid_accuracy": 0.9842076897621155, "metrics/particle/class_4/efficiency": 0.82341068983078, "metrics/particle/class_4/pid_accuracy": 0.2928921580314636, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.6456211805343628, "metrics/particle/count/bias_mean": 0.0784313753247261, "metrics/particle/count/mae": 3.362091541290283, "metrics/particle/count/prediction_mean": 70.64379119873047, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05384019762277603, "metrics/particle/event/energy_response_mean": 0.9947370290756226, "metrics/particle/event/met_abs_error": 5.571197032928467, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04501613974571228, "metrics/particle/event/scalar_pt_response_mean": 1.0072182416915894, "metrics/particle/event/vector_pt_closure": 0.04882686957716942, "metrics/particle/matched/delta_eta_abs_mean": 0.009453978389501572, "metrics/particle/matched/delta_phi_abs_mean": 0.009598388336598873, "metrics/particle/matched/delta_r_mean": 0.015137545764446259, "metrics/particle/matched/energy_relative_abs_mean": 0.09289505332708359, "metrics/particle/matched/pid_accuracy": 0.9653906226158142, "metrics/particle/matched/pt_relative_abs_mean": 0.08741084486246109, "metrics/particle/matching/duplicate_fraction": 0.033751215785741806, "metrics/particle/matching/efficiency": 0.8274903893470764, "metrics/particle/matching/f1": 0.8270307779312134, "metrics/particle/matching/purity": 0.8265717029571533, "metrics/particle/matching/target_coverage_dr0p05": 0.8136618137359619, "metrics/particle/matching/target_coverage_dr0p10": 0.8660306334495544, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7142778038978577, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8621497750282288, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01738983951508999, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0890372097492218, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08729150146245956, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02751232124865055, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14055503904819489, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004890573211014271, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38895776867866516, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03664209693670273, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.559932827949524, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8179.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25429028272628784, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0271159410476685, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1297838687896729, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.023955225944519, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12607769668102264, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.594173431396484, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014504513703286648, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7617357969284058, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.030607279390096664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.874143123626709, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9859.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8068462014198303, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5809402465820312, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.997199535369873, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1023285388946533, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.07938303053379059, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09055285155773163, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.048023700714111, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.46544453501701355, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.763577461242676, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028180280700325966, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09606426954269409, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0982155129313469, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02616015262901783, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15024712681770325, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025510204955935478, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9850448369979858, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.27377650141716003, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2580344676971436, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1568.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8185569643974304, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4991672039031982, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21256.390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21256.21875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.277515709400177, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1071976.625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08569182455539703, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.975872755050659, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.20191515982151031, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.75694751739502, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2544.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5703892111778259, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.437281608581543, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.826298236846924, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9804528951644897, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09564264118671417, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07920430600643158, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.678322792053223, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1120086908340454, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.881669998168945, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013018722645938396, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09538760036230087, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08827206492424011, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02024834416806698, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14408700168132782, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006564993527717888, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11860489845275879, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.002445208141580224, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5285580158233643, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19802.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15056145191192627, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6190208792686462, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.49380138516426086, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3830896317958832, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0971790999174118, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.954347610473633, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003016056027263403, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.26629069447517395, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013734573498368263, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4449291229248047, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22546.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8661155700683594, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.6577857732772827, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.012982368469238, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.891842842102051, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.26080650091171265, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.339266300201416, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.14821195602417, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7176975607872009, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.728837490081787, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015033163130283356, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09366575628519058, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08852829784154892, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022572677582502365, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14345383644104004, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031473145354539156, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23941420018672943, "metrics/particle/momentum/all/accepted/pz_bias": -0.023031482473015785, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1933298110961914, "metrics/particle/momentum/all/accepted/samples": 29549.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2284475862979889, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7982106804847717, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1547.927734375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1547.81396484375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1184583231806755, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 289218.78125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012275028973817825, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6032894849777222, "metrics/particle/momentum/all/assigned/pz_bias": -0.014471649192273617, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0428879261016846, "metrics/particle/momentum/all/assigned/samples": 34949.0, "metrics/particle/momentum/all/efficiency": 0.826568603515625, "metrics/particle/momentum/all/event/energy_bias": -1.1985557079315186, "metrics/particle/momentum/all/event/p3_error_mean": 14.652976036071777, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.750251293182373, "metrics/particle/momentum/all/event/px_bias": 0.4358321726322174, "metrics/particle/momentum/all/event/py_bias": 0.5090234875679016, "metrics/particle/momentum/all/event/pz_abs_mean": 10.594430923461914, "metrics/particle/momentum/all/event/pz_bias": -0.8597556352615356, "metrics/particle/momentum/all/event/pz_residual_rms": 18.759323120117188, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008909649215638638, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05229569599032402, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.042071182280778885, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010138875804841518, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09226562827825546, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015026296023279428, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14568716287612915, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.010343567468225956, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8288080096244812, "metrics/particle/momentum/class_1/accepted/samples": 15972.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09005876630544662, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.26460129022598267, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3184534013271332, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.26116234064102173, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.049253035336732864, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.817972660064697, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006302521098405123, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.34242498874664307, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.010962615720927715, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.650615930557251, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9316922426223755, "metrics/particle/momentum/class_1/event/energy_bias": 2.8997669219970703, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.086235046386719, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.940950393676758, "metrics/particle/momentum/class_1/event/px_bias": 0.4800339341163635, "metrics/particle/momentum/class_1/event/py_bias": 0.05783954635262489, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.183794975280762, "metrics/particle/momentum/class_1/event/pz_bias": -0.2587963342666626, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.015344619750977, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03505237400531769, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20022107660770416, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20236068964004517, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.033489856868982315, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24229037761688232, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04337722808122635, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7952343225479126, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.35321667790412903, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.218667507171631, "metrics/particle/momentum/class_2/accepted/samples": 1291.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8954421877861023, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7330940365791321, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20796.583984375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20796.11328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3734615445137024, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1060369.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07884615659713745, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.047156810760498, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.21678166091442108, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.664722919464111, "metrics/particle/momentum/class_2/assigned/samples": 2600.0, "metrics/particle/momentum/class_2/efficiency": 0.44671279191970825, "metrics/particle/momentum/class_2/event/energy_bias": -9.428398132324219, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.374001502990723, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.233822822570801, "metrics/particle/momentum/class_2/event/px_bias": -0.2706809639930725, "metrics/particle/momentum/class_2/event/py_bias": 0.48884809017181396, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.653935432434082, "metrics/particle/momentum/class_2/event/pz_bias": -1.327207088470459, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.669089317321777, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021304236724972725, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1381804645061493, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13894252479076385, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0448342002928257, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18076448142528534, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008447372820228338, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1893475353717804, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009634430520236492, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7722721695899963, "metrics/particle/momentum/class_3/accepted/samples": 11838.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2728489339351654, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4017136096954346, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4390921592712402, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3213584423065186, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15347950160503387, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.085692405700684, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007558218669146299, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4759271740913391, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.013690222054719925, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4171645641326904, "metrics/particle/momentum/class_3/assigned/samples": 14686.0, "metrics/particle/momentum/class_3/efficiency": 0.7793797850608826, "metrics/particle/momentum/class_3/event/energy_bias": 7.1182451248168945, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.47118854522705, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.580416679382324, "metrics/particle/momentum/class_3/event/px_bias": 0.23199141025543213, "metrics/particle/momentum/class_3/event/py_bias": -0.1599654257297516, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.45736026763916, "metrics/particle/momentum/class_3/event/pz_bias": 0.2592605948448181, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.057122230529785, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011641446501016617, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1180625706911087, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11779617518186569, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05762030556797981, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17323866486549377, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010638297535479069, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5750242471694946, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17134200036525726, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9199062585830688, "metrics/particle/momentum/class_4/accepted/samples": 282.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2787824273109436, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4324822425842285, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2714316844940186, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.20743465423584, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1836158186197281, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.656729698181152, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7392412424087524, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12705019116401672, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2374775409698486, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7966101765632629, "metrics/particle/momentum/class_4/event/energy_bias": -1.7711516618728638, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1057863235473633, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6114859580993652, "metrics/particle/momentum/class_4/event/px_bias": -0.10659332573413849, "metrics/particle/momentum/class_4/event/py_bias": 0.141523540019989, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1065454483032227, "metrics/particle/momentum/class_4/event/pz_bias": 0.3897762596607208, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.112838268280029, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007077991031110287, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.029537471011281013, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028277238830924034, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007838345132768154, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07965438067913055, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15803678333759308, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03847549483180046, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4355236887931824, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0397118516266346, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05633515119552612, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.051788099110126495, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0008544112206436694, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15377689898014069, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.24741488695144653, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0018054756801575422, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8966479301452637, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.017018206417560577, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8177575469017029, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6634609699249268, "metrics/particle/momentum/class_5/event/px_bias": 0.10108108818531036, "metrics/particle/momentum/class_5/event/py_bias": -0.01922227442264557, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.38393068313598633, "metrics/particle/momentum/class_5/event/pz_bias": 0.07721095532178879, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.544783592224121, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4182054996490479, "step_valid_time": 5477.225467920303, "step_total_time": 5478.643673658371, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0094857538100115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08469817273845415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2920, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9583196586806695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9589490968801314, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.958634274458306, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041050903119868636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.95274040039383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533661740558292, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9530531845042679, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9583196586806695, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0079612599695857, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08932953311766567, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2280, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2159, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504385964912281, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9508556384379113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04956140350877193, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9477611940298507, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9469298245614035, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9473453268977622, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9512730465320457, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0102670007252093, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08971820779231199, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1227, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1221, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06904400606980274, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9264036418816388, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9264036418816388, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9264036418816388, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9309559939301972} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_33000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..a6f99e84dd02d7140657ddd0783365055049d755 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5630920803334596, "train": {"Classification_binary": 2.5944950580596924, "Classification": 0.017946790903806686, "Regression_pt": 0.13221220672130585, "Regression_eta": 0.012758507393300533, "Regression_sin_phi": 0.0064849560149014, "Regression_cos_phi": 0.005904126446694136, "Regression_energy": 0.1365201622247696, "Total": 7.899482727050781, "Total_unweighted": 2.9063218077644706}, "train_group": {}, "valid": {"Classification_binary": 2.6821416219075522, "Classification": 0.018853506694237392, "Regression_pt": 0.1422757605711619, "Regression_eta": 0.011964214344819387, "Regression_sin_phi": 0.006881494075059891, "Regression_cos_phi": 0.007342450320720673, "Regression_energy": 0.1481168568134308, "Total": 8.363065083821615, "Total_unweighted": 3.017575904726982, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01876986399292946, "diagnostic/loss/cld_pf/Classification_binary": 2.6803910732269287, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007022049743682146, "diagnostic/loss/cld_pf/Regression_energy": 0.14764925837516785, "diagnostic/loss/cld_pf/Regression_eta": 0.010815219953656197, "diagnostic/loss/cld_pf/Regression_pt": 0.1428903490304947, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006402772851288319, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08696839213371277, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20355750620365143, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05984910577535629, "diagnostic/regression/cld_pf/energy_residual_rms": 0.43237897753715515, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05064629390835762, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19770848751068115, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07781343162059784, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4270341694355011, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9325769543647766, "metrics/particle/class_1/pid_accuracy": 0.984648585319519, "metrics/particle/class_2/efficiency": 0.4476472735404968, "metrics/particle/class_2/pid_accuracy": 0.7347925901412964, "metrics/particle/class_3/efficiency": 0.7767229080200195, "metrics/particle/class_3/pid_accuracy": 0.9837878942489624, "metrics/particle/class_4/efficiency": 0.8062562942504883, "metrics/particle/class_4/pid_accuracy": 0.3279098868370056, "metrics/particle/class_5/efficiency": 0.9495145678520203, "metrics/particle/class_5/pid_accuracy": 0.6584867238998413, "metrics/particle/count/bias_mean": 0.3901960849761963, "metrics/particle/count/mae": 3.3457515239715576, "metrics/particle/count/prediction_mean": 70.95555877685547, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05449105054140091, "metrics/particle/event/energy_response_mean": 1.0062700510025024, "metrics/particle/event/met_abs_error": 5.917936325073242, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046003103256225586, "metrics/particle/event/scalar_pt_response_mean": 1.0135787725448608, "metrics/particle/event/vector_pt_closure": 0.04931330308318138, "metrics/particle/matched/delta_eta_abs_mean": 0.009466126561164856, "metrics/particle/matched/delta_phi_abs_mean": 0.01006592158228159, "metrics/particle/matched/delta_r_mean": 0.015483291819691658, "metrics/particle/matched/energy_relative_abs_mean": 0.09445329755544662, "metrics/particle/matched/pid_accuracy": 0.9656358957290649, "metrics/particle/matched/pt_relative_abs_mean": 0.08765502274036407, "metrics/particle/matching/duplicate_fraction": 0.035095151513814926, "metrics/particle/matching/efficiency": 0.8263881802558899, "metrics/particle/matching/f1": 0.824109673500061, "metrics/particle/matching/purity": 0.8218437433242798, "metrics/particle/matching/target_coverage_dr0p05": 0.815014123916626, "metrics/particle/matching/target_coverage_dr0p10": 0.8667716383934021, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7135090231895447, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8618996739387512, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01723192259669304, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0892576277256012, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08587141335010529, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022249694913625717, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1382739096879959, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005133846774697304, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38492798805236816, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.010118059813976288, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.546984314918518, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8181.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2672305107116699, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.212074637413025, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2257980108261108, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1158992052078247, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12745296955108643, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.076324462890625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01628565602004528, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8089436888694763, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02394144795835018, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.270801544189453, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9886.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8070434927940369, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.0331155061721802, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.010669231414795, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.155956268310547, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4870523512363434, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.40011903643608093, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.9939680099487305, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5621864795684814, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.712825775146484, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024887988343834877, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09527476131916046, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09514395892620087, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02046840637922287, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14559870958328247, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025284450501203537, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0168709754943848, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.09607076644897461, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5223841667175293, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1582.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8030204772949219, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.591314673423767, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 29740.849609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 29740.68359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2746315002441406, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1509904.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08145849406719208, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9952964782714844, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07405536621809006, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.75763988494873, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2578.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5754820108413696, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.7663857936859131, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.507455825805664, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1248207092285156, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.14815053343772888, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.03659282624721527, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.357254505157471, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.06435364484786987, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.545166015625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013935471884906292, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09742625057697296, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08918794244527817, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0220969058573246, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14510656893253326, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00035464586107991636, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1194722130894661, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0048339213244616985, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5055859684944153, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19738.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15014269948005676, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6390319466590881, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5551029443740845, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.44555601477622986, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09979615360498428, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.745294094085693, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0031020119786262512, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2690441906452179, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0006252959137782454, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4347931146621704, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22566.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8633162975311279, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.884548008441925, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.125283718109131, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.9831013679504395, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.3406487703323364, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.05202966928482056, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1924734115600586, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2754090428352356, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.574244499206543, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015436952002346516, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09504561126232147, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08858761936426163, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022051947191357613, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14327122271060944, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003016846952959895, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2412097454071045, "metrics/particle/momentum/all/accepted/pz_bias": -0.011191882193088531, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2247517108917236, "metrics/particle/momentum/all/accepted/samples": 29501.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2312345951795578, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8708356022834778, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2189.453369140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2189.33984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12046816945075989, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 409610.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012589209713041782, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.622048020362854, "metrics/particle/momentum/all/assigned/pz_bias": -0.011803866364061832, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.160651683807373, "metrics/particle/momentum/all/assigned/samples": 35030.0, "metrics/particle/momentum/all/efficiency": 0.8252258896827698, "metrics/particle/momentum/all/event/energy_bias": 2.684049367904663, "metrics/particle/momentum/all/event/p3_error_mean": 15.407697677612305, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.925422668457031, "metrics/particle/momentum/all/event/px_bias": 1.9758516550064087, "metrics/particle/momentum/all/event/py_bias": -0.3114964962005615, "metrics/particle/momentum/all/event/pz_abs_mean": 11.360612869262695, "metrics/particle/momentum/all/event/pz_bias": -0.2224237024784088, "metrics/particle/momentum/all/event/pz_residual_rms": 19.52021026611328, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008515730500221252, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05356474220752716, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04233786463737488, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0095223942771554, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09285721927881241, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015053628012537956, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1496715545654297, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.009219571948051453, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9466264843940735, "metrics/particle/momentum/class_1/accepted/samples": 15943.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08768996596336365, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3446592688560486, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4124162495136261, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3563268482685089, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.050297584384679794, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.712475299835205, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00636013550683856, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3369565010070801, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.017437126487493515, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2826454639434814, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9300006031990051, "metrics/particle/momentum/class_1/event/energy_bias": 3.411207675933838, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.904548645019531, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.975879192352295, "metrics/particle/momentum/class_1/event/px_bias": 0.9563518762588501, "metrics/particle/momentum/class_1/event/py_bias": -0.07105377316474915, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.961424350738525, "metrics/particle/momentum/class_1/event/pz_bias": -1.1512351036071777, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.518479347229004, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03698583319783211, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1946951448917389, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19662822782993317, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0398486889898777, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23736724257469177, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03953488543629646, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7397187948226929, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.24629493057727814, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9747488498687744, "metrics/particle/momentum/class_2/accepted/samples": 1290.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8999237418174744, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8430453538894653, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 29062.453125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 29061.990234375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3718726336956024, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1492635.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07922668755054474, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1717405319213867, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09754539281129837, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.884631633758545, "metrics/particle/momentum/class_2/assigned/samples": 2638.0, "metrics/particle/momentum/class_2/efficiency": 0.4463667869567871, "metrics/particle/momentum/class_2/event/energy_bias": -6.231522560119629, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.407697677612305, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.280108451843262, "metrics/particle/momentum/class_2/event/px_bias": 0.5166163444519043, "metrics/particle/momentum/class_2/event/py_bias": -0.18568891286849976, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.623165130615234, "metrics/particle/momentum/class_2/event/pz_bias": -1.1862868070602417, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.98139762878418, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022640563547611237, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14090916514396667, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1397174894809723, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04552660137414932, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18113388121128082, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000930389913264662, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19442440569400787, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007506738882511854, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8403481245040894, "metrics/particle/momentum/class_3/accepted/samples": 11823.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2816420793533325, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4958226680755615, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5209662914276123, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4013797044754028, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1566404104232788, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.748006820678711, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00814774539321661, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5022751688957214, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.006612465716898441, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.541872024536133, "metrics/particle/momentum/class_3/assigned/samples": 14728.0, "metrics/particle/momentum/class_3/efficiency": 0.7783922553062439, "metrics/particle/momentum/class_3/event/energy_bias": 6.963696479797363, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.718449592590332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.59893274307251, "metrics/particle/momentum/class_3/event/px_bias": 0.7069891691207886, "metrics/particle/momentum/class_3/event/py_bias": -0.43109971284866333, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.656646251678467, "metrics/particle/momentum/class_3/event/pz_bias": 1.508220911026001, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.050371170043945, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010967759415507317, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10329166054725647, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1043819710612297, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.040619466453790665, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15795215964317322, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010676156729459763, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5741517543792725, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17383170127868652, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0619988441467285, "metrics/particle/momentum/class_4/accepted/samples": 281.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.20075830817222595, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.9460572004318237, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.8979156613349915, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8432731032371521, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1756373941898346, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.5025463104248047, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008498583920300007, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5939235091209412, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.17177395522594452, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9552001953125, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.7937853336334229, "metrics/particle/momentum/class_4/event/energy_bias": -1.580756664276123, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1131303310394287, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6333577632904053, "metrics/particle/momentum/class_4/event/px_bias": -0.2988258898258209, "metrics/particle/momentum/class_4/event/py_bias": 0.24334782361984253, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0850074291229248, "metrics/particle/momentum/class_4/event/pz_bias": 0.5207646489143372, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8025527000427246, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007110749837011099, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023213857784867287, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021763073280453682, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0028581037186086178, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05458473786711693, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15524978935718536, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01867389865219593, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5013619661331177, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.025586290284991264, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05905068293213844, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.054765816777944565, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.023169053718447685, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1712040901184082, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.23910804092884064, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08873903006315231, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8880475759506226, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": 0.12142455577850342, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.967350423336029, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8030685186386108, "metrics/particle/momentum/class_5/event/px_bias": 0.09472034126520157, "metrics/particle/momentum/class_5/event/py_bias": 0.13299806416034698, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4261396825313568, "metrics/particle/momentum/class_5/event/pz_bias": 0.08611243963241577, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.694858193397522, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3874130249023438, "step_valid_time": 5618.425206184387, "step_total_time": 5619.812619447708, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005455948601253, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08424256773606764, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625861503117821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9638514623726585, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9632183908045977, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03614853762734144, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9586478503446012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.959907985540585, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9592775041050903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625861503117821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0056042218313963, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09521074054826761, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2285, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2163, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9514223194748359, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952881875958799, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04857768052516411, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9495171202809481, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9466083150984683, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.948060486522025, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543459174714662, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0093044088547325, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0935054143942512, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9347000759301443, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9343453510436432, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06529992406985573, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9316628701594533, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9313092979127134, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339908952959028} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_34000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..d9477554cc4a84e986a7ec6b78f11eb8965e4275 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.610458507010231, "train": {"Classification_binary": 2.6260695457458496, "Classification": 0.02100745402276516, "Regression_pt": 0.14413781464099884, "Regression_eta": 0.014947791583836079, "Regression_sin_phi": 0.007484421133995056, "Regression_cos_phi": 0.008744778111577034, "Regression_energy": 0.15334434807300568, "Total": 8.748479843139648, "Total_unweighted": 2.9757361533120275}, "train_group": {}, "valid": {"Classification_binary": 2.639439582824707, "Classification": 0.01854168251156807, "Regression_pt": 0.14093913634618124, "Regression_eta": 0.01157533253232638, "Regression_sin_phi": 0.006818901126583417, "Regression_cos_phi": 0.007182258491714795, "Regression_energy": 0.14664741357167563, "Total": 8.243014653523764, "Total_unweighted": 2.9711443074047565, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018347162753343582, "diagnostic/loss/cld_pf/Classification_binary": 2.635892391204834, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006872289814054966, "diagnostic/loss/cld_pf/Regression_energy": 0.1464172601699829, "diagnostic/loss/cld_pf/Regression_eta": 0.01062280498445034, "diagnostic/loss/cld_pf/Regression_pt": 0.14185646176338196, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006385848391801119, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08136579394340515, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2008126676082611, "diagnostic/regression/cld_pf/energy_residual_mean": 0.054246511310338974, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4269484281539917, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.044518571346998215, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19545713067054749, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07168570905923843, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4216422140598297, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9357200264930725, "metrics/particle/class_1/pid_accuracy": 0.9839218258857727, "metrics/particle/class_2/efficiency": 0.46040788292884827, "metrics/particle/class_2/pid_accuracy": 0.7569908499717712, "metrics/particle/class_3/efficiency": 0.7781293988227844, "metrics/particle/class_3/pid_accuracy": 0.9830546975135803, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.334963321685791, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.6531440019607544, "metrics/particle/count/bias_mean": 0.6339869499206543, "metrics/particle/count/mae": 3.3333332538604736, "metrics/particle/count/prediction_mean": 71.19934844970703, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.054259032011032104, "metrics/particle/event/energy_response_mean": 1.001671314239502, "metrics/particle/event/met_abs_error": 5.605968475341797, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045017167925834656, "metrics/particle/event/scalar_pt_response_mean": 1.0093300342559814, "metrics/particle/event/vector_pt_closure": 0.05021703243255615, "metrics/particle/matched/delta_eta_abs_mean": 0.008919582702219486, "metrics/particle/matched/delta_phi_abs_mean": 0.009765845723450184, "metrics/particle/matched/delta_r_mean": 0.014863289892673492, "metrics/particle/matched/energy_relative_abs_mean": 0.09441967308521271, "metrics/particle/matched/pid_accuracy": 0.9655965566635132, "metrics/particle/matched/pt_relative_abs_mean": 0.08824966102838516, "metrics/particle/matching/duplicate_fraction": 0.03418552502989769, "metrics/particle/matching/efficiency": 0.8297503590583801, "metrics/particle/matching/f1": 0.8260396718978882, "metrics/particle/matching/purity": 0.822361946105957, "metrics/particle/matching/target_coverage_dr0p05": 0.8152827024459839, "metrics/particle/matching/target_coverage_dr0p10": 0.8686981797218323, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7151762247085571, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8653637766838074, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016863370314240456, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09037452936172485, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08786658942699432, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02537844330072403, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14158132672309875, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005484460853040218, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.395447313785553, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.043194063007831573, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5498243570327759, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8205.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2612760066986084, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0301196575164795, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0571255683898926, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9481090307235718, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12577028572559357, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.42524242401123, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015254066325724125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7569413185119629, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05625050142407417, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6297547817230225, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9899.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8094110488891602, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.857956051826477, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.237393379211426, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1111974716186523, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2653404474258423, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.06410291790962219, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.302560806274414, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.3098697662353516, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.540294647216797, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026585469022393227, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09991424530744553, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10174544900655746, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.025629019364714622, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15576821565628052, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02322661690413952, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.018788456916809, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2425534874200821, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.397333860397339, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1593.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8118804097175598, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3803346157073975, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 34729.58203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 34729.4140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2803523540496826, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1774473.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08349291235208511, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1202850341796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.23867560923099518, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.681807518005371, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2611.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5794834494590759, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.34834223985671997, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.493276119232178, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1045037508010864, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.005942719057202339, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08869945257902145, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.338528633117676, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.4508720338344574, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.588674545288086, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012971795164048672, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0965299904346466, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08854055404663086, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01602885127067566, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14351671934127808, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004033274599350989, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11761438846588135, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.01310168206691742, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5034357309341431, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19835.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14946533739566803, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7060321569442749, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5722101330757141, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4559984505176544, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09728867560625076, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.235830783843994, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003677122062072158, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27329060435295105, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0020213741809129715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5434386730194092, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22572.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8675589561462402, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.3062029182910919, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.012273788452148, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.9137654304504395, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.37090229988098145, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.26683855056762695, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0732100009918213, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.08570397645235062, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.407297611236572, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014781161211431026, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09500754624605179, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08906381577253342, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019133714959025383, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14367108047008514, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030371544416993856, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2429877668619156, "metrics/particle/momentum/all/accepted/pz_bias": -0.0337686650454998, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2063089609146118, "metrics/particle/momentum/all/accepted/samples": 29633.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23031532764434814, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.847664475440979, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2585.4375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2585.319091796875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11894989013671875, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 484095.28125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012884099967777729, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6216504573822021, "metrics/particle/momentum/all/assigned/pz_bias": -0.03493621200323105, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2342774868011475, "metrics/particle/momentum/all/assigned/samples": 35082.0, "metrics/particle/momentum/all/efficiency": 0.8289182782173157, "metrics/particle/momentum/all/event/energy_bias": 0.8158168196678162, "metrics/particle/momentum/all/event/p3_error_mean": 16.078994750976562, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.074383735656738, "metrics/particle/momentum/all/event/px_bias": -0.6303000450134277, "metrics/particle/momentum/all/event/py_bias": 0.2914350926876068, "metrics/particle/momentum/all/event/pz_abs_mean": 11.93421745300293, "metrics/particle/momentum/all/event/pz_bias": -1.6750378608703613, "metrics/particle/momentum/all/event/pz_residual_rms": 21.113407135009766, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008717896416783333, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05327718332409859, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04213504493236542, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008042490109801292, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09207970649003983, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015635750023648143, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1512669324874878, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.021215133368968964, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9418554306030273, "metrics/particle/momentum/class_1/accepted/samples": 15989.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09019788354635239, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.28043586015701294, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30197983980178833, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24262183904647827, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05000875145196915, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.451859474182129, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006243799813091755, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3498843014240265, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.020961595699191093, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.744011402130127, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9326838850975037, "metrics/particle/momentum/class_1/event/energy_bias": 2.20212459564209, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.003808975219727, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.972515106201172, "metrics/particle/momentum/class_1/event/px_bias": 0.2221401035785675, "metrics/particle/momentum/class_1/event/py_bias": 0.20287159085273743, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.061373710632324, "metrics/particle/momentum/class_1/event/pz_bias": -0.5873141884803772, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.68051528930664, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036023374646902084, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19974999129772186, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20242993533611298, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06420891731977463, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24090474843978882, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04009077325463295, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7533546686172485, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.35232579708099365, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.1571269035339355, "metrics/particle/momentum/class_2/accepted/samples": 1322.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8916162252426147, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7756117582321167, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 34037.08984375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 34036.60546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37575075030326843, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1756733.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08483483642339706, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.206981658935547, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1606900840997696, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.883986473083496, "metrics/particle/momentum/class_2/assigned/samples": 2664.0, "metrics/particle/momentum/class_2/efficiency": 0.45743945240974426, "metrics/particle/momentum/class_2/event/energy_bias": -7.383011817932129, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.555837631225586, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.260256767272949, "metrics/particle/momentum/class_2/event/px_bias": -0.7796639800071716, "metrics/particle/momentum/class_2/event/py_bias": 0.07632886618375778, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.726171493530273, "metrics/particle/momentum/class_2/event/pz_bias": -1.9965639114379883, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.710515975952148, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02076791040599346, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14015264809131622, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14001920819282532, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04272400215268135, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18153777718544006, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007580224191769958, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1919986605644226, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.017685845494270325, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6558369398117065, "metrics/particle/momentum/class_3/accepted/samples": 11873.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2740330994129181, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.507490634918213, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4755420684814453, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3530977964401245, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15243323147296906, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.70933723449707, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007726718205958605, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4731173515319824, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.02943876013159752, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1893434524536133, "metrics/particle/momentum/class_3/assigned/samples": 14754.0, "metrics/particle/momentum/class_3/efficiency": 0.7816841006278992, "metrics/particle/momentum/class_3/event/energy_bias": 7.2422590255737305, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.39120101928711, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.22135066986084, "metrics/particle/momentum/class_3/event/px_bias": 0.1872451901435852, "metrics/particle/momentum/class_3/event/py_bias": -0.11084457486867905, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.677534103393555, "metrics/particle/momentum/class_3/event/pz_bias": 0.47673845291137695, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.78624153137207, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010984733700752258, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11052393168210983, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11090120673179626, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05106693133711815, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16264183819293976, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010638297535479069, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5584167838096619, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05942235141992569, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.829310655593872, "metrics/particle/momentum/class_4/accepted/samples": 282.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3133319616317749, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7405418157577515, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6139123439788818, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.543839931488037, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16949152946472168, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.956539154052734, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6978243589401245, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.02111375704407692, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0417704582214355, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7966101765632629, "metrics/particle/momentum/class_4/event/energy_bias": -1.3472189903259277, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.976367473602295, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.52609121799469, "metrics/particle/momentum/class_4/event/px_bias": -0.3772122859954834, "metrics/particle/momentum/class_4/event/py_bias": 0.10473785549402237, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0228309631347656, "metrics/particle/momentum/class_4/event/pz_bias": 0.34332722425460815, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.6543262004852295, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007914658635854721, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025391221046447754, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02512776292860508, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009696184657514095, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.061534542590379715, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16072246432304382, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01470902282744646, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4939659535884857, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.028508946299552917, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04645571857690811, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04676701873540878, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0004343214677646756, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1379198580980301, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.24261212348937988, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.06629924476146698, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9629215598106384, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.10166387259960175, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.92057865858078, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7623989582061768, "metrics/particle/momentum/class_5/event/px_bias": 0.1171908974647522, "metrics/particle/momentum/class_5/event/py_bias": 0.01834135875105858, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.40455788373947144, "metrics/particle/momentum/class_5/event/pz_bias": 0.08877462893724442, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6443663835525513, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4675428867340088, "step_valid_time": 5759.707976341248, "step_total_time": 5761.1755194664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052297633613598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0810235711880809, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3050, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9619297669839186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.960983606557377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9614564539937674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03901639344262295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9554098360655737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9558799409545679, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9619297669839186, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045858018882927, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09154046140769312, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9508556384379113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9510642966864165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04914436156208864, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9460052677787533, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9455901711276876, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.945797673908273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9512730465320457, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006489113464458, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08998159090493152, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0637329286798179, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936267071320182} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_35000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..aaeef8f0c117b67d84b1a7fabd8a124ebb0634f5 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6578249336870026, "train": {"Classification_binary": 2.665592670440674, "Classification": 0.018682969734072685, "Regression_pt": 0.13446033000946045, "Regression_eta": 0.011007515713572502, "Regression_sin_phi": 0.007539569400250912, "Regression_cos_phi": 0.00805444736033678, "Regression_energy": 0.14041633903980255, "Total": 8.228865623474121, "Total_unweighted": 2.9857538416981697}, "train_group": {}, "valid": {"Classification_binary": 2.6454997062683105, "Classification": 0.018687378615140915, "Regression_pt": 0.14277939001719156, "Regression_eta": 0.011821825057268143, "Regression_sin_phi": 0.0068570468574762344, "Regression_cos_phi": 0.0072584133595228195, "Regression_energy": 0.14850486318270364, "Total": 8.316277821858725, "Total_unweighted": 2.981408623357614, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018594516441226006, "diagnostic/loss/cld_pf/Classification_binary": 2.64341139793396, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006959284655749798, "diagnostic/loss/cld_pf/Regression_energy": 0.14788192510604858, "diagnostic/loss/cld_pf/Regression_eta": 0.010740854777395725, "diagnostic/loss/cld_pf/Regression_pt": 0.143233522772789, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006401941180229187, "diagnostic/regression/cld_pf/energy_pred_mean": 0.06746232509613037, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19820930063724518, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04034304991364479, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42484745383262634, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.029838725924491882, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19180205464363098, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0570058599114418, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41856828331947327, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9368315935134888, "metrics/particle/class_1/pid_accuracy": 0.9852297306060791, "metrics/particle/class_2/efficiency": 0.4496980607509613, "metrics/particle/class_2/pid_accuracy": 0.7514567971229553, "metrics/particle/class_3/efficiency": 0.7848540544509888, "metrics/particle/class_3/pid_accuracy": 0.9830318689346313, "metrics/particle/class_4/efficiency": 0.8395559787750244, "metrics/particle/class_4/pid_accuracy": 0.32451921701431274, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6383838653564453, "metrics/particle/count/bias_mean": 0.22483660280704498, "metrics/particle/count/mae": 3.4235293865203857, "metrics/particle/count/prediction_mean": 70.79019927978516, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05438905954360962, "metrics/particle/event/energy_response_mean": 0.9856345057487488, "metrics/particle/event/met_abs_error": 5.455940246582031, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04453512281179428, "metrics/particle/event/scalar_pt_response_mean": 0.9962968826293945, "metrics/particle/event/vector_pt_closure": 0.04912447929382324, "metrics/particle/matched/delta_eta_abs_mean": 0.00868699699640274, "metrics/particle/matched/delta_phi_abs_mean": 0.010401112027466297, "metrics/particle/matched/delta_r_mean": 0.01518701110035181, "metrics/particle/matched/energy_relative_abs_mean": 0.09308930486440659, "metrics/particle/matched/pid_accuracy": 0.966062068939209, "metrics/particle/matched/pt_relative_abs_mean": 0.08602393418550491, "metrics/particle/matching/duplicate_fraction": 0.03340442478656769, "metrics/particle/matching/efficiency": 0.8323993682861328, "metrics/particle/matching/f1": 0.8310753703117371, "metrics/particle/matching/purity": 0.8297556042671204, "metrics/particle/matching/target_coverage_dr0p05": 0.8161718845367432, "metrics/particle/matching/target_coverage_dr0p10": 0.8674848079681396, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7222989201545715, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8670958280563354, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01724555715918541, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08778666704893112, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08386074751615524, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008946972899138927, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13537617027759552, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00557643361389637, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3988877236843109, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01680702716112137, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6286046504974365, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8249.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26944512128829956, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9167979955673218, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0070009231567383, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8865438103675842, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11771277338266373, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.102005958557129, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013812716118991375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7386189103126526, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04123731330037117, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8356857299804688, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9846.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8137515783309937, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3153014183044434, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.644459247589111, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0486645698547363, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.01839449256658554, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.15502752363681793, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.633594989776611, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4693337082862854, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.965412139892578, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026999173685908318, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09337397664785385, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0933414027094841, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006064188200980425, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1434633433818817, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024605678394436836, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9488314390182495, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18024975061416626, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.090733528137207, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1585.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8080074787139893, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.418422818183899, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11694.609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11694.42578125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2685692012310028, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 591333.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07935887575149536, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.873561143875122, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05794328823685646, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.95909595489502, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2558.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5765733122825623, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.6560852527618408, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.372560501098633, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0407731533050537, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.017500899732112885, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09706661850214005, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.2303056716918945, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.05961036682128906, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.310279846191406, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013274124823510647, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09613378345966339, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08767751604318619, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01246624905616045, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14274144172668457, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005019576638005674, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1173420324921608, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006024121306836605, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.514390766620636, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19922.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14611290395259857, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6912429332733154, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5372670888900757, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4209545850753784, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09235062450170517, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.337811470031738, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033219647593796253, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.26925209164619446, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.001065845019184053, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.624159812927246, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22577.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8713642358779907, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.3070760667324066, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.567612648010254, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.324766159057617, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.7448086738586426, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.16656599938869476, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5120487213134766, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.12111522257328033, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.44258975982666, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01510617695748806, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09367277473211288, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08692112565040588, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011149615049362183, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1407773494720459, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031926333904266357, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2396831214427948, "metrics/particle/momentum/all/accepted/pz_bias": -0.018293773755431175, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1921730041503906, "metrics/particle/momentum/all/accepted/samples": 29756.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2292281836271286, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8079045414924622, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 855.8032836914062, "metrics/particle/momentum/all/assigned/pt_relative_bias": 855.680908203125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11237528920173645, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 159906.8125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011834995821118355, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.591804563999176, "metrics/particle/momentum/all/assigned/pz_bias": -0.015156169421970844, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.136122703552246, "metrics/particle/momentum/all/assigned/samples": 34981.0, "metrics/particle/momentum/all/efficiency": 0.8323589563369751, "metrics/particle/momentum/all/event/energy_bias": -3.2784628868103027, "metrics/particle/momentum/all/event/p3_error_mean": 14.84312629699707, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.9584197998046875, "metrics/particle/momentum/all/event/px_bias": -0.7439151406288147, "metrics/particle/momentum/all/event/py_bias": -0.22452692687511444, "metrics/particle/momentum/all/event/pz_abs_mean": 10.534601211547852, "metrics/particle/momentum/all/event/pz_bias": -0.5308386087417603, "metrics/particle/momentum/all/event/pz_residual_rms": 19.754695892333984, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008630791679024696, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05355250462889671, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04124631732702255, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002465573837980628, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09146975725889206, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018732438329607248, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15475164353847504, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.016782883554697037, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9529472589492798, "metrics/particle/momentum/class_1/accepted/samples": 16015.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08928558975458145, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.28559353947639465, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.34015896916389465, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.27782192826271057, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04662970453500748, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.197727680206299, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006302888970822096, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3396434783935547, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.03487798199057579, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.660618782043457, "metrics/particle/momentum/class_1/assigned/samples": 17135.0, "metrics/particle/momentum/class_1/efficiency": 0.9342005252838135, "metrics/particle/momentum/class_1/event/energy_bias": 0.7379034161567688, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.17587661743164, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.167662143707275, "metrics/particle/momentum/class_1/event/px_bias": -0.19945025444030762, "metrics/particle/momentum/class_1/event/py_bias": 0.209213986992836, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.07618522644043, "metrics/particle/momentum/class_1/event/pz_bias": -1.4453734159469604, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.211524963378906, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03523000329732895, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20121127367019653, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20333857834339142, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03680987283587456, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24303458631038666, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03973013535141945, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7391895055770874, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.27635249495506287, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.056074619293213, "metrics/particle/momentum/class_2/accepted/samples": 1334.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8774560689926147, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7439172267913818, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11368.5234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11368.0634765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36259976029396057, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 583072.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07297605276107788, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.017421245574951, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.12167983502149582, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.349762439727783, "metrics/particle/momentum/class_2/assigned/samples": 2631.0, "metrics/particle/momentum/class_2/efficiency": 0.4615916907787323, "metrics/particle/momentum/class_2/event/energy_bias": -7.781954765319824, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.695752143859863, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.021448612213135, "metrics/particle/momentum/class_2/event/px_bias": -0.3640165627002716, "metrics/particle/momentum/class_2/event/py_bias": 0.35912615060806274, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.961337089538574, "metrics/particle/momentum/class_2/event/pz_bias": -0.3199440538883209, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.728231430053711, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021724054589867592, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13596010208129883, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.135401651263237, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027573268860578537, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1754874885082245, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006694000330753624, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1796519160270691, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0037403267342597246, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6204206347465515, "metrics/particle/momentum/class_3/accepted/samples": 11951.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2781688868999481, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4175366163253784, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3502063751220703, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2163681983947754, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.144471675157547, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.479467391967773, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007352941203862429, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.454052597284317, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.024731522426009178, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3143086433410645, "metrics/particle/momentum/class_3/assigned/samples": 14688.0, "metrics/particle/momentum/class_3/efficiency": 0.7868193984031677, "metrics/particle/momentum/class_3/event/energy_bias": 5.345468044281006, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.235004425048828, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.330021381378174, "metrics/particle/momentum/class_3/event/px_bias": -0.036001503467559814, "metrics/particle/momentum/class_3/event/py_bias": -0.9226925373077393, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.365187168121338, "metrics/particle/momentum/class_3/event/pz_bias": 0.7592983245849609, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.4441556930542, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011778579093515873, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10946745425462723, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10988417267799377, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04127268120646477, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16478003561496735, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013840830884873867, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5559638738632202, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.173621267080307, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.006528615951538, "metrics/particle/momentum/class_4/accepted/samples": 289.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2603200376033783, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.6435215473175049, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6978708505630493, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6211023330688477, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14406779408454895, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.981515884399414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6680231690406799, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1010545864701271, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2733154296875, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8163841962814331, "metrics/particle/momentum/class_4/event/energy_bias": -1.592503547668457, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.053069591522217, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5282270908355713, "metrics/particle/momentum/class_4/event/px_bias": -0.25418758392333984, "metrics/particle/momentum/class_4/event/py_bias": 0.14712244272232056, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1308724880218506, "metrics/particle/momentum/class_4/event/pz_bias": 0.34789687395095825, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.161617755889893, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007498716935515404, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028578786179423332, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027963653206825256, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0004180506512057036, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07880208641290665, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15503321588039398, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.010746288113296032, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4755217134952545, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.012934220023453236, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.045286089181900024, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04542010650038719, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.002835189225152135, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.127906933426857, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21783103048801422, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.06609245389699936, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8597008585929871, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.012624453753232956, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8737084865570068, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7146340608596802, "metrics/particle/momentum/class_5/event/px_bias": 0.10974076390266418, "metrics/particle/momentum/class_5/event/py_bias": -0.017296962440013885, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3873786926269531, "metrics/particle/momentum/class_5/event/pz_bias": 0.12728364765644073, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5617337226867676, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4023149013519287, "step_valid_time": 5900.8328766822815, "step_total_time": 5902.235191822052, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000552786135191, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08098010122990851, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2930, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2912, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616015753199869, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9625492772667543, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9620751929075685, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03745072273324573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9556941253692156, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9566360052562418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9561648333606961, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616015753199869, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9965861875086411, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09119614892398076, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2166, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9564643799472295, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556239015817223, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04353562005277045, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508340649692713, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9525065963060686, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9516695957820738, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547848990342406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998743549650441, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08468355845272924, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1229, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9317664897649734, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9321198331437239, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06823351023502654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9301972685887708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9294920394238059, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9298445202882063, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.93247344461305} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_36000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..a7e7b335447febf58b7de4e520410763b6992356 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.705191360363774, "train": {"Classification_binary": 2.7840781211853027, "Classification": 0.019043240696191788, "Regression_pt": 0.14390261471271515, "Regression_eta": 0.022785630077123642, "Regression_sin_phi": 0.006114036776125431, "Regression_cos_phi": 0.008500787429511547, "Regression_energy": 0.16095700860023499, "Total": 9.117003440856934, "Total_unweighted": 3.1453814394772053}, "train_group": {}, "valid": {"Classification_binary": 2.6473976771036782, "Classification": 0.018715382864077885, "Regression_pt": 0.14133882522583008, "Regression_eta": 0.011759391675392786, "Regression_sin_phi": 0.006804156427582105, "Regression_cos_phi": 0.007239879419406255, "Regression_energy": 0.14703319470087686, "Total": 8.277030309041342, "Total_unweighted": 2.9802885074168444, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01855945959687233, "diagnostic/loss/cld_pf/Classification_binary": 2.631913661956787, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007018419913947582, "diagnostic/loss/cld_pf/Regression_energy": 0.14806663990020752, "diagnostic/loss/cld_pf/Regression_eta": 0.010881347581744194, "diagnostic/loss/cld_pf/Regression_pt": 0.14351198077201843, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006386865861713886, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0723935216665268, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20267584919929504, "diagnostic/regression/cld_pf/energy_residual_mean": 0.045274242758750916, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4346850514411926, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03383593261241913, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19584307074546814, "diagnostic/regression/cld_pf/pt_residual_mean": 0.061003074049949646, "diagnostic/regression/cld_pf/pt_residual_rms": 0.42870116233825684, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9351834058761597, "metrics/particle/class_1/pid_accuracy": 0.9841995239257812, "metrics/particle/class_2/efficiency": 0.4451407194137573, "metrics/particle/class_2/pid_accuracy": 0.7655490040779114, "metrics/particle/class_3/efficiency": 0.7777997255325317, "metrics/particle/class_3/pid_accuracy": 0.9820020794868469, "metrics/particle/class_4/efficiency": 0.8324924111366272, "metrics/particle/class_4/pid_accuracy": 0.35878786444664, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.6653226017951965, "metrics/particle/count/bias_mean": -0.39803922176361084, "metrics/particle/count/mae": 3.27647066116333, "metrics/particle/count/prediction_mean": 70.16732025146484, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05492255836725235, "metrics/particle/event/energy_response_mean": 1.0000827312469482, "metrics/particle/event/met_abs_error": 5.57975959777832, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04538422450423241, "metrics/particle/event/scalar_pt_response_mean": 1.0075372457504272, "metrics/particle/event/vector_pt_closure": 0.04941946268081665, "metrics/particle/matched/delta_eta_abs_mean": 0.009606830775737762, "metrics/particle/matched/delta_phi_abs_mean": 0.009551609866321087, "metrics/particle/matched/delta_r_mean": 0.015235716477036476, "metrics/particle/matched/energy_relative_abs_mean": 0.09334457665681839, "metrics/particle/matched/pid_accuracy": 0.9662368297576904, "metrics/particle/matched/pt_relative_abs_mean": 0.08599840104579926, "metrics/particle/matching/duplicate_fraction": 0.03422258794307709, "metrics/particle/matching/efficiency": 0.828203558921814, "metrics/particle/matching/f1": 0.8305460214614868, "metrics/particle/matching/purity": 0.8329017758369446, "metrics/particle/matching/target_coverage_dr0p05": 0.8080766797065735, "metrics/particle/matching/target_coverage_dr0p10": 0.8614921569824219, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7197888493537903, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8639744520187378, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017601819708943367, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08777114003896713, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08371920883655548, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004821777809411287, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13532067835330963, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004383294843137264, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3782931864261627, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.008513592183589935, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4867957830429077, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8213.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2668722867965698, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9592288136482239, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0260905027389526, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9020404815673828, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11965811997652054, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.359832763671875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014245014637708664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7440904378890991, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0137249194085598, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6364920139312744, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9828.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8102002739906311, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.32687219977378845, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.305304050445557, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2911624908447266, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.23150306940078735, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.03203565254807472, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.291715145111084, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.0638619065284729, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.803666114807129, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02718937024474144, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09581498056650162, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09516429901123047, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.003545749466866255, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14684981107711792, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02299564890563488, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9846259951591492, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15737201273441315, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1932342052459717, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1609.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7968555092811584, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6139248609542847, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 28456.40234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 28456.220703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26355066895484924, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1435698.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08680282533168793, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0066077709198, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0976463109254837, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.306193351745605, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2546.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5853037238121033, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4015696048736572, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.8338775634765625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1397796869277954, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05310603231191635, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06608494371175766, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.672686576843262, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.3433914184570312, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.4675235748291, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013132242485880852, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09686651080846786, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08799993246793747, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.00924714282155037, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14299346506595612, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004549590521492064, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11587058007717133, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005935081280767918, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5040041208267212, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19782.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15526551008224487, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7130524516105652, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5713884830474854, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.45098045468330383, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09458557516336441, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.866481781005859, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003514703828841448, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2790262997150421, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.023022739216685295, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6497210264205933, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22477.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.865240752696991, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.676801323890686, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.3013787269592285, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.122376918792725, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.20702573657035828, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.44983571767807007, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2728867530822754, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.0070221032947301865, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.699165344238281, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01513624656945467, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09428603947162628, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08720172196626663, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007709544617682695, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14112210273742676, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002769896062090993, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2358916699886322, "metrics/particle/momentum/all/accepted/pz_bias": -0.0022254304494708776, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1563804149627686, "metrics/particle/momentum/all/accepted/samples": 29604.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23360930383205414, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.848286509513855, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2079.5078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2079.382080078125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11399959772825241, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 388047.3125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012625175528228283, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.609434962272644, "metrics/particle/momentum/all/assigned/pz_bias": 0.02585228718817234, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1689634323120117, "metrics/particle/momentum/all/assigned/samples": 34851.0, "metrics/particle/momentum/all/efficiency": 0.8281070590019226, "metrics/particle/momentum/all/event/energy_bias": 0.6021040678024292, "metrics/particle/momentum/all/event/p3_error_mean": 15.190669059753418, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.798094272613525, "metrics/particle/momentum/all/event/px_bias": -0.028628729283809662, "metrics/particle/momentum/all/event/py_bias": 0.5479563474655151, "metrics/particle/momentum/all/event/pz_abs_mean": 11.065347671508789, "metrics/particle/momentum/all/event/pz_bias": 1.4002313613891602, "metrics/particle/momentum/all/event/pz_residual_rms": 20.00093650817871, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008568525314331055, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.053799550980329514, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04137671738862991, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006459882017225027, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09096617996692657, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015019713900983334, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14847056567668915, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.017427919432520866, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9053122401237488, "metrics/particle/momentum/class_1/accepted/samples": 15979.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08921431750059128, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.28194862604141235, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.34809353947639465, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.28843849897384644, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04971988871693611, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.881553649902344, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006069094408303499, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3379707336425781, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.023230236023664474, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.641634702682495, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9321005940437317, "metrics/particle/momentum/class_1/event/energy_bias": 1.8193961381912231, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.096270561218262, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.934159755706787, "metrics/particle/momentum/class_1/event/px_bias": 0.6670924425125122, "metrics/particle/momentum/class_1/event/py_bias": 0.1645885705947876, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.162909507751465, "metrics/particle/momentum/class_1/event/pz_bias": -0.6218252778053284, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.134546279907227, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035102155059576035, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1961337924003601, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19898143410682678, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03871561959385872, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23895969986915588, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.039846744388341904, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7652794122695923, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1301279515028, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.066742420196533, "metrics/particle/momentum/class_2/accepted/samples": 1305.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8856264352798462, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7163728475570679, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 27884.935546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 27884.470703125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36951500177383423, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1421257.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0823710560798645, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.990546941757202, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.031215760856866837, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.395242214202881, "metrics/particle/momentum/class_2/assigned/samples": 2598.0, "metrics/particle/momentum/class_2/efficiency": 0.45155709981918335, "metrics/particle/momentum/class_2/event/energy_bias": -4.996663570404053, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.757682800292969, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.9899749755859375, "metrics/particle/momentum/class_2/event/px_bias": -0.08595550060272217, "metrics/particle/momentum/class_2/event/py_bias": 0.7508818507194519, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.98702621459961, "metrics/particle/momentum/class_2/event/pz_bias": 0.35538363456726074, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.199384689331055, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022001231089234352, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13822051882743835, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13696615397930145, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013333838433027267, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1776420921087265, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0003369839978404343, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1796138435602188, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03082723543047905, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5783012509346008, "metrics/particle/momentum/class_3/accepted/samples": 11870.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2889246642589569, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5121132135391235, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4395785331726074, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2939074039459229, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1438656598329544, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.756401062011719, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008156271651387215, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5080092549324036, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.08164616674184799, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.898838996887207, "metrics/particle/momentum/class_3/assigned/samples": 14590.0, "metrics/particle/momentum/class_3/efficiency": 0.7814866304397583, "metrics/particle/momentum/class_3/event/energy_bias": 4.68025016784668, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.043656349182129, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.22087287902832, "metrics/particle/momentum/class_3/event/px_bias": -0.42778247594833374, "metrics/particle/momentum/class_3/event/py_bias": -0.46364298462867737, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.216182708740234, "metrics/particle/momentum/class_3/event/pz_bias": 1.288430094718933, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.916573524475098, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011039411649107933, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10681280493736267, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10720988363027573, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.052979476749897, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1591992825269699, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007067137863487005, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5333022475242615, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0569780170917511, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7856581211090088, "metrics/particle/momentum/class_4/accepted/samples": 283.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.25895172357559204, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2360849380493164, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4721996784210205, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.416433334350586, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16101695597171783, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.838464736938477, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6412392258644104, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10718432813882828, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9919277429580688, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.799435019493103, "metrics/particle/momentum/class_4/event/energy_bias": -1.0045504570007324, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8235359191894531, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.397902011871338, "metrics/particle/momentum/class_4/event/px_bias": -0.32154956459999084, "metrics/particle/momentum/class_4/event/py_bias": 0.10059642046689987, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9045633673667908, "metrics/particle/momentum/class_4/event/pz_bias": 0.2420017123222351, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.2171051502227783, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006526387296617031, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02826809138059616, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.0273158997297287, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013586371205747128, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08059412986040115, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14548054337501526, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0022275655064731836, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.464415043592453, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.027771351858973503, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.10241209715604782, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.09989028424024582, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0720650777220726, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7025960683822632, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.22923195362091064, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0647856667637825, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9059081077575684, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.10367189347743988, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8057030439376831, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6428875923156738, "metrics/particle/momentum/class_5/event/px_bias": 0.13956640660762787, "metrics/particle/momentum/class_5/event/py_bias": -0.004467541351914406, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.38069912791252136, "metrics/particle/momentum/class_5/event/pz_bias": 0.13624122738838196, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6626156568527222, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.2880775928497314, "step_valid_time": 6041.544268846512, "step_total_time": 6042.832346916199, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015845752751853, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08278959368250582, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2909, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650511045169798, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9628289473684211, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03494889548302011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547095503774204, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9591163864160897, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9569078947368421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9606170003281916, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011254667259786, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09043016616438215, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2177, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2164, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569230769230769, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9562925543597628, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043076923076923075, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9499561018437226, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9512087912087912, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9505820338238524, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9556628621597892, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0039364249008564, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08682565012754828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1228, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9383561643835616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9369300911854104, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06164383561643835, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9317147192716236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9345509893455098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9331306990881459, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9355083459787557} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_37000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..6a5d6a9d400af3d3242bd0915162fad04a57c385 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7525577870405458, "train": {"Classification_binary": 2.645902156829834, "Classification": 0.017878875136375427, "Regression_pt": 0.14598502218723297, "Regression_eta": 0.013938486576080322, "Regression_sin_phi": 0.00810454972088337, "Regression_cos_phi": 0.007104609161615372, "Regression_energy": 0.15303684771060944, "Total": 8.576370239257812, "Total_unweighted": 2.991950547322631}, "train_group": {}, "valid": {"Classification_binary": 2.6188810666402182, "Classification": 0.018432994683583576, "Regression_pt": 0.13881335655848184, "Regression_eta": 0.011521914352973303, "Regression_sin_phi": 0.006703378011782964, "Regression_cos_phi": 0.0071280406167109804, "Regression_energy": 0.14463056127230325, "Total": 8.154221216837565, "Total_unweighted": 2.946111312136054, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018280455842614174, "diagnostic/loss/cld_pf/Classification_binary": 2.6140944957733154, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006844916380941868, "diagnostic/loss/cld_pf/Regression_energy": 0.14459942281246185, "diagnostic/loss/cld_pf/Regression_eta": 0.01076459139585495, "diagnostic/loss/cld_pf/Regression_pt": 0.13961264491081238, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006256976630538702, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0758473202586174, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19854781031608582, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04872804135084152, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4278331398963928, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.0403837151825428, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19182434678077698, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06755085289478302, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4211091995239258, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9371765851974487, "metrics/particle/class_1/pid_accuracy": 0.9849284291267395, "metrics/particle/class_2/efficiency": 0.461661159992218, "metrics/particle/class_2/pid_accuracy": 0.7522211074829102, "metrics/particle/class_3/efficiency": 0.7808763980865479, "metrics/particle/class_3/pid_accuracy": 0.9834801554679871, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.33740830421447754, "metrics/particle/class_5/efficiency": 0.9592233300209045, "metrics/particle/class_5/pid_accuracy": 0.6255060434341431, "metrics/particle/count/bias_mean": 0.2150326818227768, "metrics/particle/count/mae": 3.3052287101745605, "metrics/particle/count/prediction_mean": 70.7803955078125, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05226791650056839, "metrics/particle/event/energy_response_mean": 0.990180253982544, "metrics/particle/event/met_abs_error": 5.232012748718262, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04349543899297714, "metrics/particle/event/scalar_pt_response_mean": 1.0018624067306519, "metrics/particle/event/vector_pt_closure": 0.04663762450218201, "metrics/particle/matched/delta_eta_abs_mean": 0.008852853439748287, "metrics/particle/matched/delta_phi_abs_mean": 0.009530172683298588, "metrics/particle/matched/delta_r_mean": 0.014638184569776058, "metrics/particle/matched/energy_relative_abs_mean": 0.0928613618016243, "metrics/particle/matched/pid_accuracy": 0.9659788012504578, "metrics/particle/matched/pt_relative_abs_mean": 0.08597533404827118, "metrics/particle/matching/duplicate_fraction": 0.03389846161007881, "metrics/particle/matching/efficiency": 0.8317232728004456, "metrics/particle/matching/f1": 0.8304579257965088, "metrics/particle/matching/purity": 0.8291964530944824, "metrics/particle/matching/target_coverage_dr0p05": 0.8165053725242615, "metrics/particle/matching/target_coverage_dr0p10": 0.8675033450126648, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7224748730659485, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8675218820571899, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016493719071149826, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08714716881513596, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08414541184902191, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019442973658442497, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13625751435756683, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004979353863745928, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38383638858795166, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01641141064465046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.548256278038025, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8234.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25975751876831055, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9654335379600525, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0186138153076172, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9106273055076599, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12089023739099503, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.556282997131348, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01375821977853775, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7410669326782227, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.034821681678295135, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.823789119720459, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9885.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8122718930244446, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.37964606285095215, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.837006092071533, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9702138900756836, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.0247315913438797, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.10536154359579086, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.9033002853393555, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.6142212748527527, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.61379623413086, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025386983528733253, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09568220376968384, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09675312787294388, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.022587288171052933, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14924249053001404, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027431420981884003, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0055296421051025, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.22893792390823364, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.390124559402466, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1604.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7667484879493713, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6004478931427002, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 19698.126953125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 19697.96484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25881442427635193, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1000569.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0840759426355362, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.835970878601074, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2051875740289688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.403960227966309, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2581.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5834848880767822, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2585225105285645, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.568443775177002, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1069334745407104, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04923229664564133, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09514987468719482, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.41916561126709, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.7707997560501099, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.69533920288086, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012890856713056564, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09629809111356735, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08749838918447495, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016247063875198364, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1427558958530426, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004526707634795457, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11780355870723724, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.014856033027172089, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5160391330718994, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19882.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14899076521396637, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7116814255714417, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5831100344657898, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46986955404281616, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09237250685691833, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.4848456382751465, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0030598670709878206, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.26829230785369873, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004241241607815027, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.590222716331482, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22550.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8696146607398987, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.05256267637014389, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.029246807098389, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.914395332336426, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.24390044808387756, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.1339455544948578, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1272785663604736, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.11053232103586197, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.568069934844971, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01456345897167921, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09372956305742264, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08706891536712646, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01747468300163746, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14134757220745087, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031628531869500875, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23941954970359802, "metrics/particle/momentum/all/accepted/pz_bias": -0.026841038838028908, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2093579769134521, "metrics/particle/momentum/all/accepted/samples": 29720.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22579456865787506, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8488258123397827, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1452.5953369140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1452.480224609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11269134283065796, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 271648.90625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012051633559167385, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5910177826881409, "metrics/particle/momentum/all/assigned/pz_bias": -0.02768567018210888, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0141913890838623, "metrics/particle/momentum/all/assigned/samples": 35016.0, "metrics/particle/momentum/all/efficiency": 0.8313519358634949, "metrics/particle/momentum/all/event/energy_bias": -1.8263137340545654, "metrics/particle/momentum/all/event/p3_error_mean": 14.77307415008545, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.61021614074707, "metrics/particle/momentum/all/event/px_bias": -0.21939973533153534, "metrics/particle/momentum/all/event/py_bias": -0.14415724575519562, "metrics/particle/momentum/all/event/pz_abs_mean": 10.671110153198242, "metrics/particle/momentum/all/event/pz_bias": -2.274488925933838, "metrics/particle/momentum/all/event/pz_residual_rms": 19.619735717773438, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00905491691082716, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.052798934280872345, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04049065709114075, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006500761024653912, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08914779126644135, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016222624108195305, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14756663143634796, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.025309104472398758, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9245635271072388, "metrics/particle/momentum/class_1/accepted/samples": 16027.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0838833898305893, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30807730555534363, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.36609312891960144, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3099834620952606, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04604879021644592, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.735819339752197, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005836348980665207, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.32488834857940674, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.039052560925483704, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4826536178588867, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.9349005222320557, "metrics/particle/momentum/class_1/event/energy_bias": 1.4206162691116333, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.641960144042969, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.75180721282959, "metrics/particle/momentum/class_1/event/px_bias": 0.23319590091705322, "metrics/particle/momentum/class_1/event/py_bias": 0.045886412262916565, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.80910062789917, "metrics/particle/momentum/class_1/event/pz_bias": -1.5973364114761353, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.62752342224121, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035180289298295975, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19757385551929474, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19931690394878387, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.048870328813791275, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23778970539569855, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04188481718301773, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7764886617660522, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2953093647956848, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.197850704193115, "metrics/particle/momentum/class_2/accepted/samples": 1337.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8695746064186096, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7181493639945984, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 19226.580078125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 19226.111328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3509833514690399, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 988577.0625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07564296573400497, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9031708240509033, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.254912406206131, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.3724822998046875, "metrics/particle/momentum/class_2/assigned/samples": 2644.0, "metrics/particle/momentum/class_2/efficiency": 0.4626297652721405, "metrics/particle/momentum/class_2/event/energy_bias": -7.596194267272949, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.826414108276367, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.026589870452881, "metrics/particle/momentum/class_2/event/px_bias": -0.49132513999938965, "metrics/particle/momentum/class_2/event/py_bias": 0.09875019639730453, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.069475173950195, "metrics/particle/momentum/class_2/event/pz_bias": -2.098001480102539, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.595259666442871, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01984376832842827, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13784603774547577, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13756611943244934, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03909078612923622, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17935419082641602, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008399831713177264, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1856440156698227, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0017432625172659755, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6693114638328552, "metrics/particle/momentum/class_3/accepted/samples": 11905.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27626293897628784, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.495896816253662, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5098857879638672, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3875759840011597, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14723676443099976, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.911173820495605, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008021208457648754, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4879232347011566, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.021109499037265778, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6530251502990723, "metrics/particle/momentum/class_3/assigned/samples": 14711.0, "metrics/particle/momentum/class_3/efficiency": 0.7837908864021301, "metrics/particle/momentum/class_3/event/energy_bias": 6.069609642028809, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.077579498291016, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.213850021362305, "metrics/particle/momentum/class_3/event/px_bias": 0.16499781608581543, "metrics/particle/momentum/class_3/event/py_bias": -0.5770310163497925, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.253083229064941, "metrics/particle/momentum/class_3/event/pz_bias": 0.9295175075531006, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.752349853515625, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011723506264388561, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1066662147641182, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10831614583730698, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04877208545804024, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1592387855052948, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00699300691485405, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4872502088546753, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11260005831718445, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7022442817687988, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27482256293296814, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.500805377960205, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.446109652519226, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3783878087997437, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.068487167358398, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6585450172424316, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2274683713912964, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0416910648345947, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.6152198314666748, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0100717544555664, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5810967683792114, "metrics/particle/momentum/class_4/event/px_bias": -0.1912097930908203, "metrics/particle/momentum/class_4/event/py_bias": 0.24711081385612488, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9837565422058105, "metrics/particle/momentum/class_4/event/pz_bias": 0.3932475447654724, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5101418495178223, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006508168298751116, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022510189563035965, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02154364436864853, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007120497524738312, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.060181405395269394, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1568921059370041, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05277608335018158, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5180127024650574, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04981417581439018, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04436923936009407, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.044837649911642075, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0020226987544447184, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13696452975273132, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2398507297039032, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.10053253173828125, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8911723494529724, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": -0.10512600839138031, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7438209652900696, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6081558465957642, "metrics/particle/momentum/class_5/event/px_bias": 0.06494151800870895, "metrics/particle/momentum/class_5/event/py_bias": 0.04112638160586357, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.32821714878082275, "metrics/particle/momentum/class_5/event/pz_bias": 0.09808443486690521, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.3772011995315552, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3150615692138672, "step_valid_time": 6182.53404545784, "step_total_time": 6183.849107265472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018040161958117, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08050750805296714, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2910, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650511045169798, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9628289473684211, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03494889548302011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9550377420413522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9594460929772503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9572368421052632, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9606170003281916, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019281554988955, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09038944541573268, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2282, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526730937773883, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9535087719298245, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04732690622261174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.948729184925504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9495614035087719, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543459174714662, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026172510350417, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0871826768136208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9364118092354277, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9374763167866617, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0635881907645723, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348978046934141, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9359605911330049, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385432473444613} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_38000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..d6281124f84c5dcd2be0cb47766cac4f49dc0afc --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.7999242137173173, "train": {"Classification_binary": 2.68359375, "Classification": 0.01737181469798088, "Regression_pt": 0.1327151209115982, "Regression_eta": 0.0073021710850298405, "Regression_sin_phi": 0.00641050236299634, "Regression_cos_phi": 0.00640494842082262, "Regression_energy": 0.13331709802150726, "Total": 7.720003128051758, "Total_unweighted": 2.987115405499935}, "train_group": {}, "valid": {"Classification_binary": 2.619484265645345, "Classification": 0.01862114171187083, "Regression_pt": 0.13835503657658896, "Regression_eta": 0.011656500399112701, "Regression_sin_phi": 0.0067835599184036255, "Regression_cos_phi": 0.007220788548390071, "Regression_energy": 0.14389464259147644, "Total": 8.168025970458984, "Total_unweighted": 2.9460159353911877, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018713518977165222, "diagnostic/loss/cld_pf/Classification_binary": 2.613879442214966, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006881734821945429, "diagnostic/loss/cld_pf/Regression_energy": 0.14442561566829681, "diagnostic/loss/cld_pf/Regression_eta": 0.010688122361898422, "diagnostic/loss/cld_pf/Regression_pt": 0.13939660787582397, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006390128284692764, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07705382257699966, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19843488931655884, "diagnostic/regression/cld_pf/energy_residual_mean": 0.049934543669223785, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4265035092830658, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04287021607160568, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19393591582775116, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0700373575091362, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4214603304862976, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9353175759315491, "metrics/particle/class_1/pid_accuracy": 0.9847345352172852, "metrics/particle/class_2/efficiency": 0.45436936616897583, "metrics/particle/class_2/pid_accuracy": 0.7457371950149536, "metrics/particle/class_3/efficiency": 0.7846342921257019, "metrics/particle/class_3/pid_accuracy": 0.9833632111549377, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.3325183391571045, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6868686676025391, "metrics/particle/count/bias_mean": 0.6973856091499329, "metrics/particle/count/mae": 3.301307201385498, "metrics/particle/count/prediction_mean": 71.26274871826172, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05218569561839104, "metrics/particle/event/energy_response_mean": 0.9959036111831665, "metrics/particle/event/met_abs_error": 5.087101936340332, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04389221593737602, "metrics/particle/event/scalar_pt_response_mean": 1.0078163146972656, "metrics/particle/event/vector_pt_closure": 0.045992013067007065, "metrics/particle/matched/delta_eta_abs_mean": 0.009297365322709084, "metrics/particle/matched/delta_phi_abs_mean": 0.00958167016506195, "metrics/particle/matched/delta_r_mean": 0.015021034516394138, "metrics/particle/matched/energy_relative_abs_mean": 0.0918581411242485, "metrics/particle/matched/pid_accuracy": 0.965994119644165, "metrics/particle/matched/pt_relative_abs_mean": 0.08691515028476715, "metrics/particle/matching/duplicate_fraction": 0.03390747681260109, "metrics/particle/matching/efficiency": 0.8318251371383667, "metrics/particle/matching/f1": 0.8277349472045898, "metrics/particle/matching/purity": 0.823684811592102, "metrics/particle/matching/target_coverage_dr0p05": 0.8141990303993225, "metrics/particle/matching/target_coverage_dr0p10": 0.8677904605865479, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7210577726364136, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8679942488670349, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01745227351784706, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08757849782705307, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08615470677614212, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020773224532604218, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.138591006398201, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005218446720391512, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.39124685525894165, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0031501580961048603, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5788538455963135, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8240.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2645142078399658, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0890679359436035, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1281341314315796, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0154850482940674, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12602077424526215, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.647419929504395, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014215142466127872, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7580615878105164, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0005055521614849567, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.976579427719116, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9919.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8128637671470642, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.1619744300842285, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.148181438446045, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.140648365020752, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.053588464856147766, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.07645957171916962, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.176189422607422, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5738147497177124, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.677193641662598, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02595055289566517, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09591381251811981, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09737596660852432, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017142362892627716, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14862056076526642, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025355596095323563, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0008656978607178, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.03064320608973503, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2555675506591797, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1617.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7918790578842163, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5451635122299194, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 25965.056640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 25964.880859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27098503708839417, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1326083.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08125718683004379, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8852803707122803, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0005626276251859963, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.297492980957031, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2609.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5882139205932617, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.5825990438461304, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.553849697113037, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1354494094848633, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.013502158224582672, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.19048184156417847, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.415220260620117, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.5816113948822021, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.93305015563965, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01305444911122322, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09455928951501846, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08820280432701111, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01753365993499756, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1428423523902893, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004534233594313264, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11519759893417358, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.001243061851710081, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.47700607776641846, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19849.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15449917316436768, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6101372241973877, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4894469976425171, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3760838806629181, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09473823755979538, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.187704563140869, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002391708781942725, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2549518644809723, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.011547881178557873, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4851269721984863, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22578.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8681712746620178, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.2039109170436859, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.6630072593688965, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.532509803771973, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.22770217061042786, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.054610058665275574, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0909504890441895, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.06495247781276703, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.229201316833496, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014976318925619125, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0926966592669487, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0881340280175209, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01841096580028534, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1419990360736847, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003130680648609996, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23997950553894043, "metrics/particle/momentum/all/accepted/pz_bias": 3.6382709367899224e-05, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.191813349723816, "metrics/particle/momentum/all/accepted/samples": 29706.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23295196890830994, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8149453997612, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1930.298828125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1930.1812744140625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11667521297931671, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 361507.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011593460105359554, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5925828218460083, "metrics/particle/momentum/all/assigned/pz_bias": 0.007527906447649002, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0064122676849365, "metrics/particle/momentum/all/assigned/samples": 35106.0, "metrics/particle/momentum/all/efficiency": 0.8309603333473206, "metrics/particle/momentum/all/event/energy_bias": -0.2167135775089264, "metrics/particle/momentum/all/event/p3_error_mean": 13.591104507446289, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.958482265472412, "metrics/particle/momentum/all/event/px_bias": -0.18761584162712097, "metrics/particle/momentum/all/event/py_bias": 0.05941222608089447, "metrics/particle/momentum/all/event/pz_abs_mean": 9.98065185546875, "metrics/particle/momentum/all/event/pz_bias": 1.072749137878418, "metrics/particle/momentum/all/event/pz_residual_rms": 17.440671920776367, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008648364804685116, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0516238696873188, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04243224114179611, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012730908580124378, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0921490415930748, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015021594008430839, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14857353270053864, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00021713845490012318, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8795625567436218, "metrics/particle/momentum/class_1/accepted/samples": 15977.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08962728828191757, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.31007474660873413, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3781108260154724, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3253513276576996, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04866378754377365, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.4575300216674805, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006185085978358984, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3280775845050812, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.00430208770558238, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2964584827423096, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9319838881492615, "metrics/particle/momentum/class_1/event/energy_bias": 3.824927806854248, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.729061126708984, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.856112480163574, "metrics/particle/momentum/class_1/event/px_bias": 0.20335625112056732, "metrics/particle/momentum/class_1/event/py_bias": 0.2150769829750061, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.926198482513428, "metrics/particle/momentum/class_1/event/pz_bias": -0.1766311526298523, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.04876708984375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03500008210539818, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1989968717098236, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20079313218593597, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04349688068032265, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23936991393566132, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03978978842496872, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7489527463912964, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14164449274539948, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.115291118621826, "metrics/particle/momentum/class_2/accepted/samples": 1332.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8955816030502319, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7906874418258667, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 25379.234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 25378.75390625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36905208230018616, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1311093.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07568378001451492, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.083189010620117, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.024758828803896904, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.4565229415893555, "metrics/particle/momentum/class_2/assigned/samples": 2669.0, "metrics/particle/momentum/class_2/efficiency": 0.4608996510505676, "metrics/particle/momentum/class_2/event/energy_bias": -8.920531272888184, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.82927131652832, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.863945960998535, "metrics/particle/momentum/class_2/event/px_bias": -0.29860585927963257, "metrics/particle/momentum/class_2/event/py_bias": 0.43449491262435913, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.314336776733398, "metrics/particle/momentum/class_2/event/pz_bias": -0.03341633081436157, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.119940757751465, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021390292793512344, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1362072080373764, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13693198561668396, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.031848739832639694, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17782488465309143, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011721366317942739, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18862707912921906, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.011818917468190193, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7347420454025269, "metrics/particle/momentum/class_3/accepted/samples": 11944.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.28109851479530334, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3767322301864624, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3926440477371216, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2624772787094116, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1501387655735016, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.860980987548828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006566032767295837, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4544921815395355, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.024588927626609802, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2899093627929688, "metrics/particle/momentum/class_3/assigned/samples": 14773.0, "metrics/particle/momentum/class_3/efficiency": 0.7863585352897644, "metrics/particle/momentum/class_3/event/energy_bias": 6.061593055725098, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.29407787322998, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.240349292755127, "metrics/particle/momentum/class_3/event/px_bias": -0.042287468910217285, "metrics/particle/momentum/class_3/event/py_bias": -0.7282160520553589, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.461602210998535, "metrics/particle/momentum/class_3/event/pz_bias": 0.6894468665122986, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.988698959350586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012001958675682545, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11359808593988419, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11476074159145355, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06356916576623917, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17238737642765045, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006968641187995672, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5062140822410583, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.16105104982852936, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6571338176727295, "metrics/particle/momentum/class_4/accepted/samples": 287.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2744472026824951, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.370014190673828, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3640353679656982, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3081657886505127, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.66840934753418, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5607909560203552, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12638124823570251, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6267261505126953, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8107344508171082, "metrics/particle/momentum/class_4/event/energy_bias": -1.3761844635009766, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8125076293945312, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3694255352020264, "metrics/particle/momentum/class_4/event/px_bias": -0.19515585899353027, "metrics/particle/momentum/class_4/event/py_bias": 0.12416766583919525, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9613693952560425, "metrics/particle/momentum/class_4/event/pz_bias": 0.44582125544548035, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.345313549041748, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006995788309723139, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026066208258271217, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025667840614914894, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01690729334950447, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06839866191148758, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16398216784000397, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.035142600536346436, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4932377338409424, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.010749460197985172, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04429464414715767, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04398158937692642, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.022301379591226578, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023255813866853714, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12321728467941284, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.22589479386806488, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.022709229961037636, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8615697026252747, "metrics/particle/momentum/class_5/assigned/samples": 172.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.19348067045211792, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8206354379653931, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6706364154815674, "metrics/particle/momentum/class_5/event/px_bias": 0.14507706463336945, "metrics/particle/momentum/class_5/event/py_bias": 0.013888683170080185, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3651718497276306, "metrics/particle/momentum/class_5/event/pz_bias": 0.1475285440683365, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.585617184638977, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4622750282287598, "step_valid_time": 6322.906666994095, "step_total_time": 6324.368942260742, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0077269567857114, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07628981801345369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2940, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9648834919593042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9648834919593042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9648834919593042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035116508040695764, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9648834919593042, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0061375235896466, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08947251857472938, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04521510096575944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547848990342406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008143685680764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08675040412187884, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9301972685887708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.932319391634981, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9312571211545765, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06768060836501902, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9279210925644916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9300380228136882, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9289783516900874, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9301972685887708} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_39000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..ae13144e90dbcd55fea77e6ce6cd407fb194f933 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.8472906403940887, "train": {"Classification_binary": 2.4544365406036377, "Classification": 0.019446002319455147, "Regression_pt": 0.12468656897544861, "Regression_eta": 0.009785924106836319, "Regression_sin_phi": 0.006318454630672932, "Regression_cos_phi": 0.005792759358882904, "Regression_energy": 0.12576057016849518, "Total": 7.453249931335449, "Total_unweighted": 2.746226820163429}, "train_group": {}, "valid": {"Classification_binary": 2.6044154167175293, "Classification": 0.018258058776458103, "Regression_pt": 0.13629637161890665, "Regression_eta": 0.011486756304899851, "Regression_sin_phi": 0.0067174552629391355, "Regression_cos_phi": 0.007093468680977821, "Regression_energy": 0.14213417967160544, "Total": 8.073492685953775, "Total_unweighted": 2.926401707033316, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018220262601971626, "diagnostic/loss/cld_pf/Classification_binary": 2.5981812477111816, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006835205014795065, "diagnostic/loss/cld_pf/Regression_energy": 0.14205585420131683, "diagnostic/loss/cld_pf/Regression_eta": 0.010658279992640018, "diagnostic/loss/cld_pf/Regression_pt": 0.13681042194366455, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006298868916928768, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08617457002401352, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1980116367340088, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05905528739094734, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42524585127830505, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05148746818304062, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19283021986484528, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07865460216999054, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4193921387195587, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9352025985717773, "metrics/particle/class_1/pid_accuracy": 0.9857777953147888, "metrics/particle/class_2/efficiency": 0.45402756333351135, "metrics/particle/class_2/pid_accuracy": 0.7432873249053955, "metrics/particle/class_3/efficiency": 0.7749428749084473, "metrics/particle/class_3/pid_accuracy": 0.9837507009506226, "metrics/particle/class_4/efficiency": 0.8183652758598328, "metrics/particle/class_4/pid_accuracy": 0.3329223096370697, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.6551724076271057, "metrics/particle/count/bias_mean": -0.1196078434586525, "metrics/particle/count/mae": 3.2411763668060303, "metrics/particle/count/prediction_mean": 70.44575500488281, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0557655468583107, "metrics/particle/event/energy_response_mean": 1.006230354309082, "metrics/particle/event/met_abs_error": 5.403131484985352, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04748361185193062, "metrics/particle/event/scalar_pt_response_mean": 1.0168626308441162, "metrics/particle/event/vector_pt_closure": 0.04781169444322586, "metrics/particle/matched/delta_eta_abs_mean": 0.008393828757107258, "metrics/particle/matched/delta_phi_abs_mean": 0.009528040885925293, "metrics/particle/matched/delta_r_mean": 0.014241131953895092, "metrics/particle/matched/energy_relative_abs_mean": 0.0919385552406311, "metrics/particle/matched/pid_accuracy": 0.9664126038551331, "metrics/particle/matched/pt_relative_abs_mean": 0.08656051009893417, "metrics/particle/matching/duplicate_fraction": 0.034365665167570114, "metrics/particle/matching/efficiency": 0.8275737762451172, "metrics/particle/matching/f1": 0.828275740146637, "metrics/particle/matching/purity": 0.8289788365364075, "metrics/particle/matching/target_coverage_dr0p05": 0.8161997199058533, "metrics/particle/matching/target_coverage_dr0p10": 0.8661510944366455, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.717927098274231, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8627240061759949, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016286427155137062, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0875401496887207, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08539355546236038, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02684428170323372, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1381574273109436, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003911024425178766, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37467554211616516, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.014037379063665867, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4426054954528809, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8182.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2573302090167999, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0782501697540283, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0797044038772583, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9793709516525269, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12292257696390152, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.690409660339355, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0148966358974576, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7697119116783142, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0664333775639534, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.797396659851074, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9868.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8071421384811401, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.2278926372528076, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.531408309936523, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.331864833831787, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.33360111713409424, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.013697825372219086, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.51654052734375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0107940435409546, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.494178771972656, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025153405964374542, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09693154692649841, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.099270299077034, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02968105487525463, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1516052782535553, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026791276410222054, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9935140609741211, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12651152908802032, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.096588611602783, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1605.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7774722576141357, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.559237003326416, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 18704.484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 18704.3125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2807154059410095, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 948450.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08553654700517654, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9078636169433594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.013052636757493019, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.325575828552246, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2572.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5838486552238464, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.4012656211853027, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.370543479919434, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0656771659851074, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03296634182333946, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07790204137563705, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.2276506423950195, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.32697609066963196, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.00047492980957, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012332512065768242, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09463738650083542, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08777593076229095, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02381492778658867, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14275872707366943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004047763650305569, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11644771695137024, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008597555570304394, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5212374329566956, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19764.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1463886946439743, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6347821354866028, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5145228505134583, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4090406596660614, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09733333438634872, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.050950527191162, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002933333395048976, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2648155093193054, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.024803385138511658, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.468203067779541, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22500.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8644534945487976, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.0666840076446533, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.370800018310547, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.084238052368164, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.30420833826065063, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.22529229521751404, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.464506149291992, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0459654331207275, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.729574203491211, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014123601838946342, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09279691427946091, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08774059265851974, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02497229166328907, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14199860394001007, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002808703575283289, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2355811446905136, "metrics/particle/momentum/all/accepted/pz_bias": 0.0027655556332319975, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.130804419517517, "metrics/particle/momentum/all/accepted/samples": 29551.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22417685389518738, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.828080415725708, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1377.5089111328125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1377.400146484375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11805953085422516, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 257328.96875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012392672710120678, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6019716858863831, "metrics/particle/momentum/all/assigned/pz_bias": 0.03569582477211952, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.949690103530884, "metrics/particle/momentum/all/assigned/samples": 34940.0, "metrics/particle/momentum/all/efficiency": 0.8266245126724243, "metrics/particle/momentum/all/event/energy_bias": 2.893310785293579, "metrics/particle/momentum/all/event/p3_error_mean": 15.097797393798828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.607837200164795, "metrics/particle/momentum/all/event/px_bias": -0.003573596477508545, "metrics/particle/momentum/all/event/py_bias": -0.1336924433708191, "metrics/particle/momentum/all/event/pz_abs_mean": 11.118669509887695, "metrics/particle/momentum/all/event/pz_bias": 2.3837356567382812, "metrics/particle/momentum/all/event/pz_residual_rms": 19.192995071411133, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008372675627470016, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05125195160508156, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.041450247168540955, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012569800950586796, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09020140022039413, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013142249081283808, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14312168955802917, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.001260210177861154, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8306691646575928, "metrics/particle/momentum/class_1/accepted/samples": 15979.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08566639572381973, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24109552800655365, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2746008634567261, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2215435951948166, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0493755117058754, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.6888372898101807, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0053694406524300575, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3203028738498688, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01946750283241272, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.424835681915283, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.9321005940437317, "metrics/particle/momentum/class_1/event/energy_bias": 3.8110618591308594, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.858323097229004, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.849496841430664, "metrics/particle/momentum/class_1/event/px_bias": 0.22206197679042816, "metrics/particle/momentum/class_1/event/py_bias": -0.15694671869277954, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.99068546295166, "metrics/particle/momentum/class_1/event/pz_bias": 0.7128593325614929, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.728927612304688, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0343090184032917, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.197824627161026, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19988740980625153, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03287867084145546, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2403264194726944, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03874813765287399, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7168259620666504, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.21744313836097717, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9583370685577393, "metrics/particle/momentum/class_2/accepted/samples": 1342.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8411589860916138, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6823180317878723, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 18373.93359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 18373.4765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36631014943122864, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 940081.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07868602126836777, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.954608201980591, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06155082955956459, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.589196681976318, "metrics/particle/momentum/class_2/assigned/samples": 2618.0, "metrics/particle/momentum/class_2/efficiency": 0.464359849691391, "metrics/particle/momentum/class_2/event/energy_bias": -6.373461723327637, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.904890060424805, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.205660820007324, "metrics/particle/momentum/class_2/event/px_bias": -0.3196556270122528, "metrics/particle/momentum/class_2/event/py_bias": 0.24415603280067444, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.086511611938477, "metrics/particle/momentum/class_2/event/pz_bias": -0.1923411786556244, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.708320617675781, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01978244073688984, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13781438767910004, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13817520439624786, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04765421152114868, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17971710860729218, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006791747873649001, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1877044141292572, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.027661781758069992, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6524644494056702, "metrics/particle/momentum/class_3/accepted/samples": 11779.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27737897634506226, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5304619073867798, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4942803382873535, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.379670262336731, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15401405096054077, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.962359428405762, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00907168723642826, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5161001086235046, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07164920121431351, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4568288326263428, "metrics/particle/momentum/class_3/assigned/samples": 14661.0, "metrics/particle/momentum/class_3/efficiency": 0.7754954099655151, "metrics/particle/momentum/class_3/event/energy_bias": 6.675326347351074, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.401488304138184, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.456565856933594, "metrics/particle/momentum/class_3/event/px_bias": 0.1986306607723236, "metrics/particle/momentum/class_3/event/py_bias": -0.33698713779449463, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.52079963684082, "metrics/particle/momentum/class_3/event/pz_bias": 1.5096430778503418, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.313088417053223, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012125801295042038, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10786780714988708, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10832028836011887, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06250026077032089, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1604812741279602, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006968641187995672, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4705875515937805, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08368927985429764, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5412518978118896, "metrics/particle/momentum/class_4/accepted/samples": 287.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.26563990116119385, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.6032335758209229, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.647342324256897, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5968828201293945, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1525423675775528, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.140362739562988, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.578917384147644, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0871579647064209, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6544322967529297, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8107344508171082, "metrics/particle/momentum/class_4/event/energy_bias": -1.2343565225601196, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.767432451248169, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.324313998222351, "metrics/particle/momentum/class_4/event/px_bias": -0.24467337131500244, "metrics/particle/momentum/class_4/event/py_bias": 0.1082705706357956, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9360610842704773, "metrics/particle/momentum/class_4/event/pz_bias": 0.3171711266040802, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.2448973655700684, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0063377078622579575, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02153998799622059, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021866755560040474, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012011900544166565, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05537037551403046, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15065540373325348, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.02164505422115326, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.48450031876564026, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.012062100693583488, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05921474099159241, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06006947532296181, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03674266114830971, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04624277353286743, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2022291123867035, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.22067822515964508, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.03761085495352745, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8554973602294922, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": 0.014740955084562302, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7441545724868774, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6092071533203125, "metrics/particle/momentum/class_5/event/px_bias": 0.14006279408931732, "metrics/particle/momentum/class_5/event/py_bias": 0.00781482644379139, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.32995977997779846, "metrics/particle/momentum/class_5/event/pz_bias": 0.036403313279151917, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4880006313323975, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3865704536437988, "step_valid_time": 6464.162983179092, "step_total_time": 6465.549554109573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009462745237805, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07828028836137668, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2926, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9602888086642599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9613931329061935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9585526315789473, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574503039264005, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9602888086642599, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0126434897503969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08499488019228763, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2287, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9505902929602099, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524644030668127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04940970703979012, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9466550065588106, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9485213581599123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543459174714662, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.013474139650716, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09218427867305379, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1316, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.939209726443769, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9384965831435079, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060790273556231005, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347496206373292, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9361702127659575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.935459377372817, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377845220030349} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_4000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..e6daa3610542bccf9fc5aab6fd1b29c9f9853336 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.189465706707086, "train": {"Classification_binary": 3.9891161918640137, "Classification": 0.029402466490864754, "Regression_pt": 0.2506580650806427, "Regression_eta": 0.010554091073572636, "Regression_sin_phi": 0.00841366034001112, "Regression_cos_phi": 0.008192560635507107, "Regression_energy": 0.2513001263141632, "Total": 12.6203031539917, "Total_unweighted": 4.547637161798775}, "train_group": {}, "valid": {"Classification_binary": 4.020556131998698, "Classification": 0.03102821856737137, "Regression_pt": 0.22930006186167398, "Regression_eta": 0.015398401767015457, "Regression_sin_phi": 0.009443818281094233, "Regression_cos_phi": 0.009738291303316752, "Regression_energy": 0.23511799176534018, "Total": 12.69610850016276, "Total_unweighted": 4.55058291554451, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.030845757573843002, "diagnostic/loss/cld_pf/Classification_binary": 4.0126729011535645, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.009454366751015186, "diagnostic/loss/cld_pf/Regression_energy": 0.2351934015750885, "diagnostic/loss/cld_pf/Regression_eta": 0.014394710771739483, "diagnostic/loss/cld_pf/Regression_pt": 0.2300681471824646, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.008881926536560059, "diagnostic/regression/cld_pf/energy_pred_mean": 0.10694418847560883, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.25465086102485657, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07982490956783295, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5280605554580688, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.07471039146184921, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2530529201030731, "diagnostic/regression/cld_pf/pt_residual_mean": 0.10187752544879913, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5260924100875854, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9043275117874146, "metrics/particle/class_1/pid_accuracy": 0.9791039824485779, "metrics/particle/class_2/efficiency": 0.24085678160190582, "metrics/particle/class_2/pid_accuracy": 0.3396404981613159, "metrics/particle/class_3/efficiency": 0.6649305820465088, "metrics/particle/class_3/pid_accuracy": 0.9780546426773071, "metrics/particle/class_4/efficiency": 0.8728557229042053, "metrics/particle/class_4/pid_accuracy": 0.11329479515552521, "metrics/particle/class_5/efficiency": 0.9475728273391724, "metrics/particle/class_5/pid_accuracy": 0.32786884903907776, "metrics/particle/count/bias_mean": 0.07647059112787247, "metrics/particle/count/mae": 4.696078300476074, "metrics/particle/count/prediction_mean": 70.64183044433594, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.08562389016151428, "metrics/particle/event/energy_response_mean": 0.981850802898407, "metrics/particle/event/met_abs_error": 10.225757598876953, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0746278166770935, "metrics/particle/event/scalar_pt_response_mean": 0.9952449798583984, "metrics/particle/event/vector_pt_closure": 0.08276847749948502, "metrics/particle/matched/delta_eta_abs_mean": 0.01415925845503807, "metrics/particle/matched/delta_phi_abs_mean": 0.0159506406635046, "metrics/particle/matched/delta_r_mean": 0.023709435015916824, "metrics/particle/matched/energy_relative_abs_mean": 0.1003488153219223, "metrics/particle/matched/pid_accuracy": 0.9488196969032288, "metrics/particle/matched/pt_relative_abs_mean": 0.09601891785860062, "metrics/particle/matching/duplicate_fraction": 0.048750024288892746, "metrics/particle/matching/efficiency": 0.749409556388855, "metrics/particle/matching/f1": 0.7490037083625793, "metrics/particle/matching/purity": 0.7485982775688171, "metrics/particle/matching/target_coverage_dr0p05": 0.699763834476471, "metrics/particle/matching/target_coverage_dr0p10": 0.7946834564208984, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.641504168510437, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.79961097240448, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026702530682086945, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09821806848049164, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09633368253707886, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02434331178665161, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15097349882125854, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006693074945360422, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4897962808609009, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04053395986557007, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.982721209526062, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7321.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.437134712934494, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5699892044067383, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6317415237426758, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.4890483617782593, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1892692893743515, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.920432090759277, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02370975911617279, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1669845581054688, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.15643185377120972, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.288270950317383, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9785.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7222057580947876, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.41897544264793396, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.092979431152344, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.657872200012207, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6222370862960815, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.37763726711273193, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.202961921691895, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.1171412467956543, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.679731369018555, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03472619503736496, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1080990880727768, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.11137481033802032, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02315371297299862, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16444319486618042, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022894522175192833, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2091434001922607, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.09775100648403168, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.183971881866455, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1223.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.035601258277893, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7235218286514282, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 36361.58984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 36361.390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.36716538667678833, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1808016.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.099878691136837, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.6101584434509277, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07471003383398056, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.650766372680664, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2473.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.44488903880119324, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.9507509469985962, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.242201805114746, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.530381202697754, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.17742332816123962, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.010806214064359665, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.021651268005371, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.608156204223633, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.025564193725586, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02160610444843769, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1021648421883583, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09667755663394928, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.027362899854779243, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15231889486312866, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006564910290762782, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13988152146339417, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.021551327779889107, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6043410301208496, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18279.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.25054630637168884, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9529115557670593, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7645777463912964, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6218916773796082, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14617584645748138, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.28572940826416, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006522516254335642, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.43755942583084106, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.056787312030792236, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1509907245635986, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22384.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.799501359462738, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.12337589263916, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.24123764038086, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.851579666137695, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.100501537322998, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.0601754188537598, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.509649276733398, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.21490740776062, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.278290748596191, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02359532192349434, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.10135819017887115, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09725381433963776, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02634682133793831, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15252797305583954, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033180478494614363, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2841395437717438, "metrics/particle/momentum/all/accepted/pz_bias": 0.021292785182595253, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.456032395362854, "metrics/particle/momentum/all/accepted/samples": 26823.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.359293133020401, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1822235584259033, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2596.711669921875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2596.5654296875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17412389814853668, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 483073.21875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018041683360934258, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8700767159461975, "metrics/particle/momentum/all/assigned/pz_bias": 0.07554572075605392, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.8513190746307373, "metrics/particle/momentum/all/assigned/samples": 34642.0, "metrics/particle/momentum/all/efficiency": 0.7503147125244141, "metrics/particle/momentum/all/event/energy_bias": -3.6551513671875, "metrics/particle/momentum/all/event/p3_error_mean": 24.804882049560547, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.395102500915527, "metrics/particle/momentum/all/event/px_bias": -0.3008411228656769, "metrics/particle/momentum/all/event/py_bias": 2.427006483078003, "metrics/particle/momentum/all/event/pz_abs_mean": 18.089168548583984, "metrics/particle/momentum/all/event/pz_bias": 7.940205097198486, "metrics/particle/momentum/all/event/pz_residual_rms": 27.242773056030273, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01701115444302559, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05935382843017578, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.052731722593307495, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01673000305891037, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10155962407588959, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001747233560308814, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1885301172733307, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01200744416564703, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1654362678527832, "metrics/particle/momentum/class_1/accepted/samples": 15453.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13893558084964752, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.375826895236969, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5441261529922485, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.47125449776649475, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07374992966651917, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.105652809143066, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0085185831412673, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.46460020542144775, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.05003733932971954, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.696469783782959, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9014174938201904, "metrics/particle/momentum/class_1/event/energy_bias": 5.43986701965332, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.51142120361328, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.58942985534668, "metrics/particle/momentum/class_1/event/px_bias": -0.07891088724136353, "metrics/particle/momentum/class_1/event/py_bias": -0.019779831171035767, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.643835067749023, "metrics/particle/momentum/class_1/event/pz_bias": 1.4218957424163818, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.029386520385742, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.046307411044836044, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23544511198997498, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23146282136440277, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.077837735414505, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27142757177352905, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.062068965286016464, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.5983076095581055, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1767197847366333, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.833405017852783, "metrics/particle/momentum/class_2/accepted/samples": 725.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.3485383987426758, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.1528877019882202, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 36820.30859375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 36819.60546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5438165664672852, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1819456.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.11998362094163895, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.404412746429443, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.11789670586585999, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.23386001586914, "metrics/particle/momentum/class_2/assigned/samples": 2442.0, "metrics/particle/momentum/class_2/efficiency": 0.25086504220962524, "metrics/particle/momentum/class_2/event/energy_bias": -18.92195701599121, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.612499237060547, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.854475021362305, "metrics/particle/momentum/class_2/event/px_bias": -0.4584296941757202, "metrics/particle/momentum/class_2/event/py_bias": 1.351013422012329, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.888116836547852, "metrics/particle/momentum/class_2/event/pz_bias": 1.9432859420776367, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.413345336914062, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.032182637602090836, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1568531095981598, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1564052253961563, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.048795051872730255, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1992378532886505, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014750713016837835, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.25592532753944397, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.04244653880596161, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9487161636352539, "metrics/particle/momentum/class_3/accepted/samples": 10169.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4607681334018707, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.158989429473877, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0934770107269287, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9510633945465088, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2337278127670288, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.999077796936035, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012384752742946148, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7628445029258728, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.13309235870838165, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.496058225631714, "metrics/particle/momentum/class_3/assigned/samples": 14534.0, "metrics/particle/momentum/class_3/efficiency": 0.6694976687431335, "metrics/particle/momentum/class_3/event/energy_bias": 13.791487693786621, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.422052383422852, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.348773002624512, "metrics/particle/momentum/class_3/event/px_bias": 0.0789366066455841, "metrics/particle/momentum/class_3/event/py_bias": 0.7496619820594788, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.020883560180664, "metrics/particle/momentum/class_3/event/pz_bias": 4.665239334106445, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.76618003845215, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021229278296232224, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09984314441680908, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.09934122860431671, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.01944316178560257, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1497212052345276, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006451612804085016, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5779441595077515, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1841965764760971, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.5420637130737305, "metrics/particle/momentum/class_4/accepted/samples": 310.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.16982445120811462, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.8738973736763, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.944770336151123, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8431147933006287, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.10451977699995041, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.22017765045166, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7472613453865051, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2714817523956299, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.9641246795654297, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8757061958312988, "metrics/particle/momentum/class_4/event/energy_bias": -3.2431905269622803, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.572873592376709, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.5155153274536133, "metrics/particle/momentum/class_4/event/px_bias": 0.2652761936187744, "metrics/particle/momentum/class_4/event/py_bias": 0.21740135550498962, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1029162406921387, "metrics/particle/momentum/class_4/event/pz_bias": 0.059101611375808716, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.205817222595215, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01569107547402382, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.029202451929450035, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028222501277923584, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.014338241890072823, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.060905586928129196, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2570995092391968, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.15040555596351624, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5484744906425476, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08880764991044998, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05678996071219444, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05875569209456444, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.00932775717228651, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17216627299785614, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4111310541629791, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.09768704324960709, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2023111581802368, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.7213592529296875, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8851723670959473, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3485853672027588, "metrics/particle/momentum/class_5/event/px_bias": -0.10771331191062927, "metrics/particle/momentum/class_5/event/py_bias": 0.1287095993757248, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0954293012619019, "metrics/particle/momentum/class_5/event/pz_bias": -0.14931756258010864, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.222444534301758, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.073944091796875, "step_valid_time": 884.3376269340515, "step_total_time": 885.411571264267, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982210299178256, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1693516781970611, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2776, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2720, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9110600590744995, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9125575279421433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9118081786828708, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08744247205785667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8926813258943223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8941485864562788, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8934143537526688, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9110600590744995, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004243055815467, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1674729035527004, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2021, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9012291483757682, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.905602117335686, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9034103410341034, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09439788266431408, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8871817383669885, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8914865460961623, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8893289328932893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9012291483757682, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038459536263868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16473286712372492, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1300, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1138, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1120, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8634294385432474, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8753846153846154, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8693659281894576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12461538461538461, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.849772382397572, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8615384615384616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8556149732620321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8634294385432474} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_40000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..a388e36fd875b153958acc6cf60f875292b66e51 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.8946570670708602, "train": {"Classification_binary": 2.599729061126709, "Classification": 0.017437413334846497, "Regression_pt": 0.14677129685878754, "Regression_eta": 0.016142411157488823, "Regression_sin_phi": 0.00554856238886714, "Regression_cos_phi": 0.004952270537614822, "Regression_energy": 0.14692823588848114, "Total": 8.172844886779785, "Total_unweighted": 2.937509251292795}, "train_group": {}, "valid": {"Classification_binary": 2.6017624537150064, "Classification": 0.018390440692504246, "Regression_pt": 0.1373742620150248, "Regression_eta": 0.011377165714899698, "Regression_sin_phi": 0.006678602347771327, "Regression_cos_phi": 0.007098592817783356, "Regression_energy": 0.14294076959292093, "Total": 8.089216868082682, "Total_unweighted": 2.9256222868959108, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01837850920855999, "diagnostic/loss/cld_pf/Classification_binary": 2.5991029739379883, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006839692126959562, "diagnostic/loss/cld_pf/Regression_energy": 0.14325040578842163, "diagnostic/loss/cld_pf/Regression_eta": 0.01061016134917736, "diagnostic/loss/cld_pf/Regression_pt": 0.13839343190193176, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006304284557700157, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08226775377988815, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19862258434295654, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05514848232269287, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42601436376571655, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04500420764088631, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19236081838607788, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07217134535312653, "diagnostic/regression/cld_pf/pt_residual_rms": 0.419953852891922, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9357200264930725, "metrics/particle/class_1/pid_accuracy": 0.9852941036224365, "metrics/particle/class_2/efficiency": 0.4547111690044403, "metrics/particle/class_2/pid_accuracy": 0.7614632844924927, "metrics/particle/class_3/efficiency": 0.7786128520965576, "metrics/particle/class_3/pid_accuracy": 0.9836861491203308, "metrics/particle/class_4/efficiency": 0.8213925361633301, "metrics/particle/class_4/pid_accuracy": 0.3488943576812744, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.6470588445663452, "metrics/particle/count/bias_mean": -0.15490196645259857, "metrics/particle/count/mae": 3.2660131454467773, "metrics/particle/count/prediction_mean": 70.41045379638672, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0510195754468441, "metrics/particle/event/energy_response_mean": 0.9957910180091858, "metrics/particle/event/met_abs_error": 5.497037887573242, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04367324709892273, "metrics/particle/event/scalar_pt_response_mean": 1.0069031715393066, "metrics/particle/event/vector_pt_closure": 0.047712940722703934, "metrics/particle/matched/delta_eta_abs_mean": 0.00831623375415802, "metrics/particle/matched/delta_phi_abs_mean": 0.009505778551101685, "metrics/particle/matched/delta_r_mean": 0.014179419726133347, "metrics/particle/matched/energy_relative_abs_mean": 0.09229101240634918, "metrics/particle/matched/pid_accuracy": 0.9670358896255493, "metrics/particle/matched/pt_relative_abs_mean": 0.08564947545528412, "metrics/particle/matching/duplicate_fraction": 0.03402086719870567, "metrics/particle/matching/efficiency": 0.8294540047645569, "metrics/particle/matching/f1": 0.8303653597831726, "metrics/particle/matching/purity": 0.8312788009643555, "metrics/particle/matching/target_coverage_dr0p05": 0.8181540369987488, "metrics/particle/matching/target_coverage_dr0p10": 0.8666975498199463, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7207891345024109, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8641689419746399, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015930691733956337, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08687979727983475, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08351833373308182, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020851299166679382, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13548201322555542, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004871513694524765, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.384542316198349, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.009511073119938374, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5765575170516968, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8211.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24858038127422333, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9803745746612549, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1134382486343384, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0069866180419922, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12129516899585724, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.171621322631836, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013601299375295639, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7170451283454895, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03730860725045204, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.482468605041504, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9852.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8100029826164246, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7970401644706726, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.0149102210998535, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6702957153320312, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.07521604746580124, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.16695308685302734, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.184225082397461, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2962353527545929, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.949103355407715, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024851929396390915, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09983933717012405, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10105857253074646, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0253014974296093, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15505364537239075, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02741432934999466, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0821897983551025, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.24048662185668945, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5380687713623047, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1605.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7850188612937927, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.894638180732727, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12314.4677734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12314.3046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2724084258079529, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 623598.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08378800004720688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0335631370544434, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.11810530722141266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.479032516479492, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2566.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5838486552238464, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2175564765930176, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.319967269897461, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.101857304573059, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0705990344285965, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.030862273648381233, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.16517448425293, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2111029028892517, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.181589126586914, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012506231665611267, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09509512037038803, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08686452358961105, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017509549856185913, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14145280420780182, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004037141625303775, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1170852780342102, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.007037718314677477, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5190966725349426, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19816.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1467674970626831, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7048739194869995, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5762185454368591, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46593785285949707, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09496063739061356, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.871341705322266, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028465951327234507, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2532857656478882, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02248304896056652, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1610947847366333, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22483.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.866727888584137, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7884293794631958, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.923094272613525, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.868756294250488, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.709382951259613, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.968899130821228, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.052680015563965, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8943548798561096, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.266616344451904, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014123844914138317, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09307563304901123, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08670610934495926, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018857590854167938, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14060276746749878, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031047516968101263, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24347171187400818, "metrics/particle/momentum/all/accepted/pz_bias": -0.00568392826244235, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2437700033187866, "metrics/particle/momentum/all/accepted/samples": 29632.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22243334352970123, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8701175451278687, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 906.0728149414062, "metrics/particle/momentum/all/assigned/pt_relative_bias": 905.9598388671875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11544081568717957, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 169088.71875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011833472177386284, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5886098742485046, "metrics/particle/momentum/all/assigned/pz_bias": 0.01633169688284397, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.035491466522217, "metrics/particle/momentum/all/assigned/samples": 34901.0, "metrics/particle/momentum/all/efficiency": 0.828890323638916, "metrics/particle/momentum/all/event/energy_bias": -0.6320868730545044, "metrics/particle/momentum/all/event/p3_error_mean": 14.33798599243164, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.561267852783203, "metrics/particle/momentum/all/event/px_bias": 0.5635677576065063, "metrics/particle/momentum/all/event/py_bias": 0.7710837721824646, "metrics/particle/momentum/all/event/pz_abs_mean": 10.309283256530762, "metrics/particle/momentum/all/event/pz_bias": 1.4016931056976318, "metrics/particle/momentum/all/event/pz_residual_rms": 20.361692428588867, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008405452594161034, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.053150027990341187, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0416850820183754, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009359207935631275, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0922081246972084, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001562011893838644, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14876073598861694, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0010806918144226074, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9012628793716431, "metrics/particle/momentum/class_1/accepted/samples": 16005.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08604705333709717, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2626427710056305, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3236311674118042, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.26683807373046875, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.049416568130254745, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.852397918701172, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0053675612434744835, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.32255786657333374, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.013090712949633598, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.713432788848877, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9336172342300415, "metrics/particle/momentum/class_1/event/energy_bias": 1.9958767890930176, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.998767852783203, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.993533134460449, "metrics/particle/momentum/class_1/event/px_bias": 0.5180383920669556, "metrics/particle/momentum/class_1/event/py_bias": 0.32170766592025757, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.034889221191406, "metrics/particle/momentum/class_1/event/pz_bias": 0.4573054611682892, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.979751586914062, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03464303910732269, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1938244104385376, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19607038795948029, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04351123049855232, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2356976717710495, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.042488619685173035, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8126006126403809, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.23760326206684113, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.401073455810547, "metrics/particle/momentum/class_2/accepted/samples": 1318.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8608208298683167, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.753250241279602, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12178.8154296875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12178.3642578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.361603707075119, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 620224.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0828835740685463, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.041757583618164, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0735924169421196, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.750466346740723, "metrics/particle/momentum/class_2/assigned/samples": 2594.0, "metrics/particle/momentum/class_2/efficiency": 0.4560553729534149, "metrics/particle/momentum/class_2/event/energy_bias": -7.11355447769165, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.232287406921387, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.360268592834473, "metrics/particle/momentum/class_2/event/px_bias": 0.13097284734249115, "metrics/particle/momentum/class_2/event/py_bias": 0.5351021885871887, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.342172622680664, "metrics/particle/momentum/class_2/event/pz_bias": -0.4585811495780945, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.992585182189941, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019707517698407173, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1362800896167755, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1355821192264557, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0379195399582386, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1761755645275116, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007589173037558794, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1915065050125122, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.008361628279089928, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7377779483795166, "metrics/particle/momentum/class_3/accepted/samples": 11859.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27070948481559753, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.582377314567566, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6885526180267334, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5671226978302002, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14870218932628632, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.67881202697754, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006967213004827499, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4674142003059387, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03082130290567875, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.269077777862549, "metrics/particle/momentum/class_3/assigned/samples": 14640.0, "metrics/particle/momentum/class_3/efficiency": 0.7807623744010925, "metrics/particle/momentum/class_3/event/energy_bias": 5.704504013061523, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.97766399383545, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.134361267089844, "metrics/particle/momentum/class_3/event/px_bias": -0.028242111206054688, "metrics/particle/momentum/class_3/event/py_bias": -0.2640124559402466, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.2332611083984375, "metrics/particle/momentum/class_3/event/pz_bias": 0.9193264245986938, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.355663299560547, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012303135357797146, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11159557104110718, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11221013963222504, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.055471573024988174, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16482758522033691, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007042253389954567, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5268164873123169, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.21095964312553406, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.689577579498291, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24891221523284912, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.086570978164673, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.050494909286499, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9933757781982422, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17514124512672424, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.776464462280273, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.687016487121582, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2643826901912689, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1659884452819824, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.2350540161132812, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9197351932525635, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4616024494171143, "metrics/particle/momentum/class_4/event/px_bias": -0.19586467742919922, "metrics/particle/momentum/class_4/event/py_bias": 0.11509452760219574, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0131337642669678, "metrics/particle/momentum/class_4/event/pz_bias": 0.42715635895729065, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.3604795932769775, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006767661310732365, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024408375844359398, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023788226768374443, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005422395188361406, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06523625552654266, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1442098766565323, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.017821302637457848, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5176057815551758, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02328563667833805, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04446868225932121, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0456496886909008, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.009423624724149704, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1321479231119156, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21945691108703613, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04797029867768288, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9144781827926636, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.016140632331371307, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8338617086410522, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6666720509529114, "metrics/particle/momentum/class_5/event/px_bias": 0.13866332173347473, "metrics/particle/momentum/class_5/event/py_bias": 0.06319183856248856, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.40364396572113037, "metrics/particle/momentum/class_5/event/pz_bias": 0.05648612976074219, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7770898342132568, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4329943656921387, "step_valid_time": 6605.003475427628, "step_total_time": 6606.436470031738, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006044431949192, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07983261974093647, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2926, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2913, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9602888086642599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9634507737899243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9618671926364234, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03654922621007573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9560223170331473, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9591702337833388, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9575936883629191, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9602888086642599, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045429434798723, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08705316522566542, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2283, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2184, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2178, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9587357330992098, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9566360052562418, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576847182635387, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04336399474375821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561018437225637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540078843626807, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9550537162902872, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9587357330992098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0069031528414962, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0885489094573102, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9396946564885497, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936834094368341, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06030534351145038, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9366412213740458, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337899543378996, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339908952959028} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_41000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..af608f65815b32c5bec3a43d7d5036f6d5e48c27 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9420234937476317, "train": {"Classification_binary": 2.700603485107422, "Classification": 0.022636348381638527, "Regression_pt": 0.15321828424930573, "Regression_eta": 0.01074912864714861, "Regression_sin_phi": 0.007175463251769543, "Regression_cos_phi": 0.006529134698212147, "Regression_energy": 0.155457004904747, "Total": 8.655045509338379, "Total_unweighted": 3.0563688492402434}, "train_group": {}, "valid": {"Classification_binary": 2.6046202977498374, "Classification": 0.01831094299753507, "Regression_pt": 0.13704042633374533, "Regression_eta": 0.011313781142234802, "Regression_sin_phi": 0.006654713302850723, "Regression_cos_phi": 0.007100772112607956, "Regression_energy": 0.14256059130032858, "Total": 8.076011657714844, "Total_unweighted": 2.92760152493914, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.0181634109467268, "diagnostic/loss/cld_pf/Classification_binary": 2.5975165367126465, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006879337597638369, "diagnostic/loss/cld_pf/Regression_energy": 0.14219476282596588, "diagnostic/loss/cld_pf/Regression_eta": 0.010641583241522312, "diagnostic/loss/cld_pf/Regression_pt": 0.13726958632469177, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006291398778557777, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08327466249465942, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19883233308792114, "diagnostic/regression/cld_pf/energy_residual_mean": 0.056155379861593246, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4294019937515259, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.0436992384493351, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1919824331998825, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07086637616157532, "diagnostic/regression/cld_pf/pt_residual_rms": 0.42215752601623535, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9352025985717773, "metrics/particle/class_1/pid_accuracy": 0.9847326874732971, "metrics/particle/class_2/efficiency": 0.46496525406837463, "metrics/particle/class_2/pid_accuracy": 0.7640284299850464, "metrics/particle/class_3/efficiency": 0.7800193428993225, "metrics/particle/class_3/pid_accuracy": 0.9837155342102051, "metrics/particle/class_4/efficiency": 0.8173562288284302, "metrics/particle/class_4/pid_accuracy": 0.3444444537162781, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.6673387289047241, "metrics/particle/count/bias_mean": -0.09869281202554703, "metrics/particle/count/mae": 3.324836492538452, "metrics/particle/count/prediction_mean": 70.46666717529297, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.052544329315423965, "metrics/particle/event/energy_response_mean": 0.9995847344398499, "metrics/particle/event/met_abs_error": 5.3741888999938965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043219905346632004, "metrics/particle/event/scalar_pt_response_mean": 1.0072243213653564, "metrics/particle/event/vector_pt_closure": 0.04749617353081703, "metrics/particle/matched/delta_eta_abs_mean": 0.008514521643519402, "metrics/particle/matched/delta_phi_abs_mean": 0.00984141044318676, "metrics/particle/matched/delta_r_mean": 0.014620997942984104, "metrics/particle/matched/energy_relative_abs_mean": 0.09289158135652542, "metrics/particle/matched/pid_accuracy": 0.9667477011680603, "metrics/particle/matched/pt_relative_abs_mean": 0.08566129207611084, "metrics/particle/matching/duplicate_fraction": 0.03426271304488182, "metrics/particle/matching/efficiency": 0.830621063709259, "metrics/particle/matching/f1": 0.8312022686004639, "metrics/particle/matching/purity": 0.8317843675613403, "metrics/particle/matching/target_coverage_dr0p05": 0.816931426525116, "metrics/particle/matching/target_coverage_dr0p10": 0.8666512370109558, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7222710847854614, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8652896881103516, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01658959873020649, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08829589188098907, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08410626649856567, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017989974468946457, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13565610349178314, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004985408391803503, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3869105279445648, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008109930902719498, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5256298780441284, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8224.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2684389054775238, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0142512321472168, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0489368438720703, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9436913132667542, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12186707556247711, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.530841827392578, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016032470390200615, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7833774089813232, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.014552558772265911, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8191978931427, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9855.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8112853765487671, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8950338363647461, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.974788188934326, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1653780937194824, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.03404214233160019, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.21908795833587646, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.977114677429199, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.17920853197574615, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.605158805847168, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025352660566568375, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09553594142198563, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09498778730630875, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016973847523331642, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.147840678691864, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023720350116491318, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9356840252876282, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.11096315830945969, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.982006788253784, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1602.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.760604739189148, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6495893001556396, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21322.400390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21322.23046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2678641080856323, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1078897.625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08238968998193741, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9268550872802734, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0026579280383884907, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.651915550231934, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2561.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5827573537826538, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0802582502365112, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.385400295257568, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.068512201309204, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.005628667771816254, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.021817736327648163, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.2394280433654785, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.09950557351112366, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.541271209716797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012664470821619034, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09589465707540512, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0874149277806282, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016887478530406952, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1422552615404129, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006039863219484687, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11908122152090073, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0012403747532516718, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5403143167495728, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19868.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14641167223453522, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6840935349464417, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5723597407341003, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4636717736721039, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09289932996034622, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.207762241363525, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002575602848082781, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2657985985279083, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.00274528656154871, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5337717533111572, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22519.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8690023422241211, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.3511916399002075, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.904085159301758, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.7263336181640625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.08453207463026047, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7153265476226807, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.186809778213501, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1733490228652954, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.244345664978027, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014436098746955395, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09377076476812363, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08690712600946426, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01719748228788376, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14076946675777435, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030645921360701323, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23731473088264465, "metrics/particle/momentum/all/accepted/pz_bias": -0.007402683608233929, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1487876176834106, "metrics/particle/momentum/all/accepted/samples": 29694.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22585999965667725, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8480075597763062, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1563.7581787109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1563.64599609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11389723420143127, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 292115.53125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012222698889672756, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6068806648254395, "metrics/particle/momentum/all/assigned/pz_bias": 0.005679966416209936, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0407121181488037, "metrics/particle/momentum/all/assigned/samples": 34935.0, "metrics/particle/momentum/all/efficiency": 0.8306246399879456, "metrics/particle/momentum/all/event/energy_bias": 1.1659672260284424, "metrics/particle/momentum/all/event/p3_error_mean": 14.577329635620117, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.444254398345947, "metrics/particle/momentum/all/event/px_bias": 0.1242029070854187, "metrics/particle/momentum/all/event/py_bias": 0.9562321901321411, "metrics/particle/momentum/all/event/pz_abs_mean": 10.700716018676758, "metrics/particle/momentum/all/event/pz_bias": 0.45206305384635925, "metrics/particle/momentum/all/event/pz_residual_rms": 19.724666595458984, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008437191136181355, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05344507843255997, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04175320267677307, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008571217767894268, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09179738909006119, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017489069141447544, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14914648234844208, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.010610192082822323, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8433519601821899, "metrics/particle/momentum/class_1/accepted/samples": 16010.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08426876366138458, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.28846368193626404, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.32614395022392273, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.27132534980773926, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04731898009777069, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.635384559631348, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005892992485314608, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.33336979150772095, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.017008205875754356, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5600335597991943, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9339088797569275, "metrics/particle/momentum/class_1/event/energy_bias": 1.6602481603622437, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.714234352111816, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.747810363769531, "metrics/particle/momentum/class_1/event/px_bias": 0.6181932687759399, "metrics/particle/momentum/class_1/event/py_bias": 0.24725258350372314, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.881580352783203, "metrics/particle/momentum/class_1/event/pz_bias": -0.6031137108802795, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.9069766998291, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03465518355369568, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1928667426109314, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19595277309417725, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04838396981358528, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23426297307014465, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03831705451011658, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7103763818740845, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.19079473614692688, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.041701316833496, "metrics/particle/momentum/class_2/accepted/samples": 1331.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.857249915599823, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7604061365127563, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20808.55859375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20808.115234375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36128050088882446, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1065867.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07926829159259796, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.056406021118164, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.019682906568050385, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.959825038909912, "metrics/particle/momentum/class_2/assigned/samples": 2624.0, "metrics/particle/momentum/class_2/efficiency": 0.4605536460876465, "metrics/particle/momentum/class_2/event/energy_bias": -5.81387186050415, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.756668090820312, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.001007556915283, "metrics/particle/momentum/class_2/event/px_bias": -0.5540260672569275, "metrics/particle/momentum/class_2/event/py_bias": 0.38300907611846924, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.053586959838867, "metrics/particle/momentum/class_2/event/pz_bias": -0.9062560200691223, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.821846961975098, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02042115107178688, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13737985491752625, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13564881682395935, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03527543321251869, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17673851549625397, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008401243248954415, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1854899376630783, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.014743850566446781, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.657350480556488, "metrics/particle/momentum/class_3/accepted/samples": 11903.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27959004044532776, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4995248317718506, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.498075008392334, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.375656008720398, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1470126360654831, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.857738494873047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007852509617805481, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4913170337677002, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.035068340599536896, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.367175817489624, "metrics/particle/momentum/class_3/assigned/samples": 14645.0, "metrics/particle/momentum/class_3/efficiency": 0.7836592197418213, "metrics/particle/momentum/class_3/event/energy_bias": 6.548003196716309, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.28809642791748, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.348012447357178, "metrics/particle/momentum/class_3/event/px_bias": 0.2076627016067505, "metrics/particle/momentum/class_3/event/py_bias": 0.07353322207927704, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.451324462890625, "metrics/particle/momentum/class_3/event/pz_bias": 1.4221789836883545, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.32045841217041, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011460003443062305, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11259031295776367, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1126435250043869, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06154954433441162, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16331073641777039, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007042253389954567, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5256450772285461, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.09744179248809814, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7539252042770386, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2780055105686188, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.023498296737671, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9673070907592773, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9102798700332642, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17231638729572296, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.547383308410645, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6652834415435791, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10353134572505951, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9788365364074707, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.3199431896209717, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8600921630859375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4200010299682617, "metrics/particle/momentum/class_4/event/px_bias": -0.2547965347766876, "metrics/particle/momentum/class_4/event/py_bias": 0.2241668701171875, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9750198125839233, "metrics/particle/momentum/class_4/event/pz_bias": 0.4222305119037628, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.591498613357544, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006821918301284313, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02927493490278721, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02843153290450573, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.002845767419785261, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08060289174318314, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15245583653450012, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.00501301558688283, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.49798154830932617, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02140113338828087, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05203982815146446, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05234494060277939, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0035858952905982733, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1500895917415619, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21326543390750885, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04996993765234947, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8580284714698792, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.09153081476688385, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8224172592163086, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6669949293136597, "metrics/particle/momentum/class_5/event/px_bias": 0.10716961324214935, "metrics/particle/momentum/class_5/event/py_bias": 0.02827046439051628, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.37155014276504517, "metrics/particle/momentum/class_5/event/pz_bias": 0.11702316254377365, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.558068871498108, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4155383110046387, "step_valid_time": 6746.457253694534, "step_total_time": 6747.8727922439575, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0072289596037216, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08046788537545135, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2919, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9635228393033191, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9628899835796387, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03647716069668091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9579914670167378, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.959250739401906, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9586206896551724, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0056167790137565, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0894429150844086, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2284, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2178, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2169, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561018437225637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9535901926444834, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9548443665059184, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046409807355516634, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9521510096575944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9496497373029772, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9508987286277948, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561018437225637, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0061332830863752, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08638256689830448, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1230, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9332321699544764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.935361216730038, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9342954804405621, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06463878326996197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9279210925644916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9300380228136882, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9289783516900874, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9332321699544764} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_42000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..548920cf1b4eecd2665d97792b9fe7f62de5d245 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9893899204244032, "train": {"Classification_binary": 2.5550732612609863, "Classification": 0.018991775810718536, "Regression_pt": 0.13804255425930023, "Regression_eta": 0.012629235163331032, "Regression_sin_phi": 0.007222576066851616, "Regression_cos_phi": 0.006802872754633427, "Regression_energy": 0.14384111762046814, "Total": 8.171082496643066, "Total_unweighted": 2.8826033929362893}, "train_group": {}, "valid": {"Classification_binary": 2.5713418324788413, "Classification": 0.018145823230346043, "Regression_pt": 0.13501888513565063, "Regression_eta": 0.011457574864228567, "Regression_sin_phi": 0.006577905888358752, "Regression_cos_phi": 0.007003946229815483, "Regression_energy": 0.14065087834993997, "Total": 7.984479904174805, "Total_unweighted": 2.8901968461771808, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01791701465845108, "diagnostic/loss/cld_pf/Classification_binary": 2.5631022453308105, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00675226841121912, "diagnostic/loss/cld_pf/Regression_energy": 0.14030864834785461, "diagnostic/loss/cld_pf/Regression_eta": 0.01052737608551979, "diagnostic/loss/cld_pf/Regression_pt": 0.13556484878063202, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006122762802988291, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08054229617118835, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19792276620864868, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05342300981283188, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4240662157535553, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04547322914004326, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1921835094690323, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07264035940170288, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41935694217681885, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9350684285163879, "metrics/particle/class_1/pid_accuracy": 0.986759603023529, "metrics/particle/class_2/efficiency": 0.45767346024513245, "metrics/particle/class_2/pid_accuracy": 0.75130695104599, "metrics/particle/class_3/efficiency": 0.7800193428993225, "metrics/particle/class_3/pid_accuracy": 0.9841663241386414, "metrics/particle/class_4/efficiency": 0.8183652758598328, "metrics/particle/class_4/pid_accuracy": 0.3304562270641327, "metrics/particle/class_5/efficiency": 0.9592233300209045, "metrics/particle/class_5/pid_accuracy": 0.6497975587844849, "metrics/particle/count/bias_mean": -0.31437909603118896, "metrics/particle/count/mae": 3.1836600303649902, "metrics/particle/count/prediction_mean": 70.25098419189453, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05203012749552727, "metrics/particle/event/energy_response_mean": 0.9933942556381226, "metrics/particle/event/met_abs_error": 5.281245708465576, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0427032932639122, "metrics/particle/event/scalar_pt_response_mean": 1.0025931596755981, "metrics/particle/event/vector_pt_closure": 0.04663589596748352, "metrics/particle/matched/delta_eta_abs_mean": 0.008245831355452538, "metrics/particle/matched/delta_phi_abs_mean": 0.00906888023018837, "metrics/particle/matched/delta_r_mean": 0.013792653568089008, "metrics/particle/matched/energy_relative_abs_mean": 0.09253335744142532, "metrics/particle/matched/pid_accuracy": 0.9673793911933899, "metrics/particle/matched/pt_relative_abs_mean": 0.08583366870880127, "metrics/particle/matching/duplicate_fraction": 0.03202337026596069, "metrics/particle/matching/efficiency": 0.8299541473388672, "metrics/particle/matching/f1": 0.8318070769309998, "metrics/particle/matching/purity": 0.8336682915687561, "metrics/particle/matching/target_coverage_dr0p05": 0.8189228177070618, "metrics/particle/matching/target_coverage_dr0p10": 0.8668179512023926, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7216875553131104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8644375205039978, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015726692974567413, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08739987760782242, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08420366793870926, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021401625126600266, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13690567016601562, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004374240525066853, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3682308793067932, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.034878261387348175, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4306706190109253, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8230.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25057077407836914, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.979385495185852, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.035124659538269, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9281492233276367, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1212090477347374, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.410664558410645, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014098793268203735, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7300318479537964, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05143725126981735, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.607820510864258, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9859.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8118773102760315, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.2936466932296753, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.8933563232421875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.97900652885437, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.0359155535697937, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.22375226020812988, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.97087287902832, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.3766276836395264, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.256638526916504, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02429855242371559, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09721694141626358, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09779141843318939, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016066567972302437, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1511346399784088, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021000618115067482, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9785114526748657, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2629515826702118, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0435688495635986, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1619.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7387369871139526, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.456243634223938, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 29609.611328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 29609.4453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26674455404281616, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1500335.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08216510713100433, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9486007690429688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3592662513256073, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.014826774597168, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2568.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5889414548873901, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.118262767791748, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.308021545410156, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0990768671035767, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.02148474007844925, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1504010260105133, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.144318103790283, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.0884952545166016, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.765106201171875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01197011023759842, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09526494890451431, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08698879182338715, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0167789738625288, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14227117598056793, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005554994568228722, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.115468330681324, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00295067741535604, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5146649479866028, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19802.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1468050628900528, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6721869707107544, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5137364268302917, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4042643904685974, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09320224076509476, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.292387962341309, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003381083719432354, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27060991525650024, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.006569034419953823, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6588994264602661, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22478.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8661155700683594, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.3617761731147766, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.503437042236328, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.512956142425537, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.1841340810060501, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.36321842670440674, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.8243725299835205, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.14858506619930267, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.8211669921875, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013685951009392738, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09318848699331284, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08680558949708939, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01802314817905426, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14130575954914093, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027317795902490616, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23274946212768555, "metrics/particle/momentum/all/accepted/pz_bias": -0.026009105145931244, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.118396282196045, "metrics/particle/momentum/all/accepted/samples": 29651.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21966302394866943, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8166398406028748, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2179.03564453125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2178.92236328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11388052999973297, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 406951.0625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012204554863274097, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.59739750623703, "metrics/particle/momentum/all/assigned/pz_bias": -0.03672988712787628, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1099812984466553, "metrics/particle/momentum/all/assigned/samples": 34905.0, "metrics/particle/momentum/all/efficiency": 0.8294218182563782, "metrics/particle/momentum/all/event/energy_bias": -1.1863923072814941, "metrics/particle/momentum/all/event/p3_error_mean": 14.712614059448242, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.523898601531982, "metrics/particle/momentum/all/event/px_bias": 0.16970328986644745, "metrics/particle/momentum/all/event/py_bias": 0.28986719250679016, "metrics/particle/momentum/all/event/pz_abs_mean": 10.795408248901367, "metrics/particle/momentum/all/event/pz_bias": -2.316538095474243, "metrics/particle/momentum/all/event/pz_residual_rms": 19.260746002197266, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008617597632110119, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05292922258377075, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.040986068546772, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007597591262310743, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.090552918612957, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014379493659362197, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14706085622310638, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.012431968934834003, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8545984625816345, "metrics/particle/momentum/class_1/accepted/samples": 15995.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08709821850061417, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2730090320110321, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3182758390903473, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2607562243938446, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04907510057091713, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.90958309173584, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005776973906904459, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3347611427307129, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.032097119837999344, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.626378059387207, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9330338835716248, "metrics/particle/momentum/class_1/event/energy_bias": 2.638988971710205, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.781373023986816, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.863918304443359, "metrics/particle/momentum/class_1/event/px_bias": 0.08404029905796051, "metrics/particle/momentum/class_1/event/py_bias": 0.05682660639286041, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.875185966491699, "metrics/particle/momentum/class_1/event/pz_bias": -1.3702372312545776, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.67378044128418, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03355041891336441, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19984079897403717, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20130549371242523, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05042390152812004, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2411433309316635, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.035874441266059875, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6953811645507812, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3242907226085663, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8900482654571533, "metrics/particle/momentum/class_2/accepted/samples": 1338.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8374025225639343, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7725938558578491, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 29187.326171875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 29186.869140625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3516314923763275, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1489642.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08023032546043396, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.058030128479004, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.34912821650505066, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.841469764709473, "metrics/particle/momentum/class_2/assigned/samples": 2605.0, "metrics/particle/momentum/class_2/efficiency": 0.46297577023506165, "metrics/particle/momentum/class_2/event/energy_bias": -7.733828544616699, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.168201446533203, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.12070894241333, "metrics/particle/momentum/class_2/event/px_bias": -0.18345895409584045, "metrics/particle/momentum/class_2/event/py_bias": -0.026034504175186157, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.482016563415527, "metrics/particle/momentum/class_2/event/pz_bias": -1.6804214715957642, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.846728324890137, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018445175141096115, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1360289454460144, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1360008865594864, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03899216651916504, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17768485844135284, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000758341746404767, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18081161379814148, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.012566047720611095, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6284266710281372, "metrics/particle/momentum/class_3/accepted/samples": 11868.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2658308446407318, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4246162176132202, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3622570037841797, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2431986331939697, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14717136323451996, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.545759201049805, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0077890134416520596, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4689614772796631, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.008475985378026962, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5357773303985596, "metrics/particle/momentum/class_3/assigned/samples": 14636.0, "metrics/particle/momentum/class_3/efficiency": 0.7813549041748047, "metrics/particle/momentum/class_3/event/energy_bias": 5.587101459503174, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.920329093933105, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.065061092376709, "metrics/particle/momentum/class_3/event/px_bias": 0.2680582106113434, "metrics/particle/momentum/class_3/event/py_bias": -0.01829782873392105, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.226664066314697, "metrics/particle/momentum/class_3/event/pz_bias": 0.3230947256088257, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.231014251708984, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011229285039007664, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10917405784130096, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11045681685209274, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.059944137930870056, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16544705629348755, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003496503457427025, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.39207908511161804, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030465319752693176, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2322638034820557, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27813607454299927, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6947309970855713, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.395752191543579, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.324260711669922, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.19098472595215, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7045966386795044, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.17552457749843597, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.019892692565918, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.6793838739395142, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9976778030395508, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5255534648895264, "metrics/particle/momentum/class_4/event/px_bias": -0.18881511688232422, "metrics/particle/momentum/class_4/event/py_bias": 0.25368908047676086, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0667332410812378, "metrics/particle/momentum/class_4/event/pz_bias": 0.309359073638916, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.104327201843262, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005818789824843407, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02150343544781208, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020148636773228645, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0027134520933032036, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.058732204139232635, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13773734867572784, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0120434844866395, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.47856852412223816, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.023938868194818497, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0522247776389122, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.047664400190114975, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.023332878947257996, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1519862860441208, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20834319293498993, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05040276423096657, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8422392010688782, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": 0.0007302537560462952, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7749031782150269, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6260704398155212, "metrics/particle/momentum/class_5/event/px_bias": 0.18987883627414703, "metrics/particle/momentum/class_5/event/py_bias": 0.023683827370405197, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3626078963279724, "metrics/particle/momentum/class_5/event/pz_bias": 0.10166695713996887, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.625347375869751, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3827733993530273, "step_valid_time": 6887.450925827026, "step_total_time": 6888.833699703217, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00298473119577, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0773057563931383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2923, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2913, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9593042336724648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9643681953150776, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9618295491938138, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03563180468492247, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9560223170331473, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9610689541405477, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9585389930898321, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9593042336724648, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002649659819494, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08855546103490297, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2280, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2181, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2173, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9574187884108868, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9565789473684211, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9569986836331724, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04342105263157895, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9539069359086918, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9530701754385965, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9574187884108868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035715566077144, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08588438271462706, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9415781487101669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9444444444444444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9430091185410334, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05555555555555555, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9421613394216134, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9407294832826748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9415781487101669} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_43000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..7f3d465a78f786b86e58c9ca403b68d0d777f532 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0367563471011745, "train": {"Classification_binary": 2.6062116622924805, "Classification": 0.018416082486510277, "Regression_pt": 0.1326366662979126, "Regression_eta": 0.010749765671789646, "Regression_sin_phi": 0.00804353691637516, "Regression_cos_phi": 0.007178084924817085, "Regression_energy": 0.13718587160110474, "Total": 8.081783294677734, "Total_unweighted": 2.92042167019099}, "train_group": {}, "valid": {"Classification_binary": 2.5466858545939126, "Classification": 0.017964211603005726, "Regression_pt": 0.13489534457524618, "Regression_eta": 0.011561719079812368, "Regression_sin_phi": 0.006652091940244039, "Regression_cos_phi": 0.006965147952238719, "Regression_energy": 0.1404868264993032, "Total": 7.958872477213542, "Total_unweighted": 2.865211196243763, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017850663512945175, "diagnostic/loss/cld_pf/Classification_binary": 2.5469236373901367, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006689370144158602, "diagnostic/loss/cld_pf/Regression_energy": 0.14084424078464508, "diagnostic/loss/cld_pf/Regression_eta": 0.010727474465966225, "diagnostic/loss/cld_pf/Regression_pt": 0.13586053252220154, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0062222834676504135, "diagnostic/regression/cld_pf/energy_pred_mean": 0.06797288358211517, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19456225633621216, "diagnostic/regression/cld_pf/energy_residual_mean": 0.040853600949048996, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42008501291275024, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.030781298875808716, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18808139860630035, "diagnostic/regression/cld_pf/pt_residual_mean": 0.05794842913746834, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41370001435279846, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9400705099105835, "metrics/particle/class_1/pid_accuracy": 0.984893262386322, "metrics/particle/class_2/efficiency": 0.4642816483974457, "metrics/particle/class_2/pid_accuracy": 0.761227011680603, "metrics/particle/class_3/efficiency": 0.7825026512145996, "metrics/particle/class_3/pid_accuracy": 0.9847502112388611, "metrics/particle/class_4/efficiency": 0.8345106244087219, "metrics/particle/class_4/pid_accuracy": 0.34703749418258667, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.6915322542190552, "metrics/particle/count/bias_mean": -0.24379085004329681, "metrics/particle/count/mae": 3.226797342300415, "metrics/particle/count/prediction_mean": 70.32157135009766, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0519135557115078, "metrics/particle/event/energy_response_mean": 0.9815812706947327, "metrics/particle/event/met_abs_error": 4.872271537780762, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04207957535982132, "metrics/particle/event/scalar_pt_response_mean": 0.9923390746116638, "metrics/particle/event/vector_pt_closure": 0.044248342514038086, "metrics/particle/matched/delta_eta_abs_mean": 0.00825515016913414, "metrics/particle/matched/delta_phi_abs_mean": 0.009037638083100319, "metrics/particle/matched/delta_r_mean": 0.013770961202681065, "metrics/particle/matched/energy_relative_abs_mean": 0.09142651408910751, "metrics/particle/matched/pid_accuracy": 0.9672425985336304, "metrics/particle/matched/pt_relative_abs_mean": 0.084491066634655, "metrics/particle/matching/duplicate_fraction": 0.03161945194005966, "metrics/particle/matching/efficiency": 0.8341221809387207, "metrics/particle/matching/f1": 0.8355655074119568, "metrics/particle/matching/purity": 0.8370139002799988, "metrics/particle/matching/target_coverage_dr0p05": 0.8194692730903625, "metrics/particle/matching/target_coverage_dr0p10": 0.8674941062927246, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7283934354782104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.869337260723114, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01545098889619112, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08630933612585068, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08308351039886475, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004709857515990734, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1342976987361908, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00557643361389637, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3960537314414978, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.027005212381482124, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6434599161148071, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8249.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25553756952285767, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.975293755531311, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9367595314979553, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8167421221733093, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11845842003822327, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.935737609863281, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013894523493945599, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7290195226669312, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.03832152485847473, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.779939651489258, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9860.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8137515783309937, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3420898914337158, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.343984603881836, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.842376232147217, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.037766143679618835, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2073667049407959, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.466042518615723, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7921355962753296, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.464829444885254, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024918166920542717, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.094087615609169, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09511467814445496, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.002700383774936199, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14570090174674988, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022825416177511215, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9764736294746399, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.23109138011932373, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2868175506591797, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1621.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7597417235374451, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7366526126861572, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 33635.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 33635.31640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2642023265361786, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1705001.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08054474741220474, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7792582511901855, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0715063214302063, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.958585262298584, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2570.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5896689891815186, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2999069690704346, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.778815746307373, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9857579469680786, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.021529488265514374, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.080288365483284, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.630051136016846, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.053338646888733, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.342150688171387, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01200741808861494, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09437783062458038, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0857887789607048, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010803276672959328, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14040611684322357, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005018317024223506, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11293473094701767, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005131226498633623, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4884527027606964, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19927.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14434602856636047, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6842931509017944, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5824095606803894, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46761786937713623, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08882368355989456, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.615079879760742, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028007468208670616, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.25274914503097534, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005929869133979082, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4362118244171143, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22494.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8715829253196716, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.8514499664306641, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.220783233642578, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.218565940856934, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.3268798589706421, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.21094384789466858, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.829918622970581, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.43447840213775635, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.989224910736084, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013663100078701973, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09212835878133774, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08554719388484955, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008675566874444485, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13903865218162537, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031211196910589933, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23829114437103271, "metrics/particle/momentum/all/accepted/pz_bias": -0.02347937971353531, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2227013111114502, "metrics/particle/momentum/all/accepted/samples": 29797.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22102440893650055, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8438920378684998, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2475.82080078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2475.699951171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11009620875120163, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 462518.71875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01165387686342001, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5731348991394043, "metrics/particle/momentum/all/assigned/pz_bias": -0.012261910364031792, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8585691452026367, "metrics/particle/momentum/all/assigned/samples": 34924.0, "metrics/particle/momentum/all/efficiency": 0.8335058093070984, "metrics/particle/momentum/all/event/energy_bias": -4.4934468269348145, "metrics/particle/momentum/all/event/p3_error_mean": 13.589811325073242, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.981983184814453, "metrics/particle/momentum/all/event/px_bias": -0.3431164622306824, "metrics/particle/momentum/all/event/py_bias": -0.3380221724510193, "metrics/particle/momentum/all/event/pz_abs_mean": 9.980925559997559, "metrics/particle/momentum/all/event/pz_bias": -1.4109959602355957, "metrics/particle/momentum/all/event/pz_residual_rms": 17.32069206237793, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008193256333470345, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05242624878883362, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04016048088669777, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0044847698882222176, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08867812901735306, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016182237304747105, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14483168721199036, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.015038183890283108, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8589204549789429, "metrics/particle/momentum/class_1/accepted/samples": 16067.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08141496777534485, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2416359782218933, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2809593081474304, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.22373849153518677, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04488152265548706, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.82699728012085, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005953075829893351, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3105739951133728, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.028557276353240013, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2616097927093506, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.9372338652610779, "metrics/particle/momentum/class_1/event/energy_bias": 0.7391115427017212, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.114028930664062, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.738962173461914, "metrics/particle/momentum/class_1/event/px_bias": 0.31178152561187744, "metrics/particle/momentum/class_1/event/py_bias": 0.20942918956279755, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.221270561218262, "metrics/particle/momentum/class_1/event/pz_bias": -1.058534860610962, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.572861671447754, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03398377075791359, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19718866050243378, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2011152058839798, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.055470071732997894, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23909200727939606, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.041267503052949905, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8127655982971191, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2708002030849457, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.324398994445801, "metrics/particle/momentum/class_2/accepted/samples": 1357.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8311446905136108, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7663213014602661, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 32953.82421875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 32953.3671875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34693098068237305, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1687688.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07548608630895615, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.94344162940979, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06513898819684982, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.5705037117004395, "metrics/particle/momentum/class_2/assigned/samples": 2623.0, "metrics/particle/momentum/class_2/efficiency": 0.46955016255378723, "metrics/particle/momentum/class_2/event/energy_bias": -8.08912467956543, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.577085494995117, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.848817348480225, "metrics/particle/momentum/class_2/event/px_bias": -0.20409232378005981, "metrics/particle/momentum/class_2/event/py_bias": 0.15814785659313202, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.93904972076416, "metrics/particle/momentum/class_2/event/pz_bias": -1.250637412071228, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.650559425354004, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01886655017733574, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13421180844306946, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13385283946990967, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.020593887194991112, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17465263605117798, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005874454509466887, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18067237734794617, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01093767024576664, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7362529635429382, "metrics/particle/momentum/class_3/accepted/samples": 11916.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2765425145626068, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5108884572982788, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.493303894996643, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.355384111404419, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14392076432704926, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.372986793518066, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006967213004827499, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4589129090309143, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.012385855428874493, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3816919326782227, "metrics/particle/momentum/class_3/assigned/samples": 14640.0, "metrics/particle/momentum/class_3/efficiency": 0.7845150828361511, "metrics/particle/momentum/class_3/event/energy_bias": 4.196006774902344, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.964041709899902, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.150458335876465, "metrics/particle/momentum/class_3/event/px_bias": -0.31485313177108765, "metrics/particle/momentum/class_3/event/py_bias": -0.952476978302002, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.169257164001465, "metrics/particle/momentum/class_3/event/pz_bias": 0.47934994101524353, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.411739349365234, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011990098282694817, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1100723147392273, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11063773185014725, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05217967554926872, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1660008430480957, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013745704665780067, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.47171705961227417, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1311582773923874, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6804068088531494, "metrics/particle/momentum/class_4/accepted/samples": 291.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2630874812602997, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.3742318153381348, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.8270411491394043, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.763767957687378, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1497175097465515, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.011308670043945, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6240674257278442, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.16414932906627655, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9446966648101807, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8220338821411133, "metrics/particle/momentum/class_4/event/energy_bias": -1.429604172706604, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8580970764160156, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4076581001281738, "metrics/particle/momentum/class_4/event/px_bias": -0.2568870186805725, "metrics/particle/momentum/class_4/event/py_bias": 0.22150325775146484, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0007189512252808, "metrics/particle/momentum/class_4/event/pz_bias": 0.3647645115852356, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5791916847229004, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006382000166922808, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023688094690442085, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022241074591875076, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006872746627777815, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.060535114258527756, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14015662670135498, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.009912917390465736, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.46341049671173096, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013203058391809464, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04595327749848366, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04589153081178665, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.011559399776160717, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13715985417366028, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20080147683620453, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0434265099465847, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8254733085632324, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.09016401320695877, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8012874126434326, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6548134088516235, "metrics/particle/momentum/class_5/event/px_bias": 0.12093445658683777, "metrics/particle/momentum/class_5/event/py_bias": 0.025374457240104675, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.36146730184555054, "metrics/particle/momentum/class_5/event/pz_bias": 0.05406177043914795, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5601845979690552, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3500211238861084, "step_valid_time": 7028.84133887291, "step_total_time": 7030.191360235214, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974979448228867, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07543104888989616, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9632425336396455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9667325428194994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9649843827059017, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033267457180500656, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9640974967061924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9623541015946079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9632425336396455, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9945249671530771, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08601055232951993, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578577699736611, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574374725756911, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576475751590959, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04256252742430891, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9543659499780606, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9545753785385122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578577699736611, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9958291745284642, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08866559889937187, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.939117199391172, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9376899696048632, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060882800608828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347496206373292, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9375951293759512, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9361702127659575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936267071320182} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_44000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..d481bb51318d68849ed8480f46b4ebd8ddb45ac7 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0841227737779464, "train": {"Classification_binary": 2.5309410095214844, "Classification": 0.016945380717515945, "Regression_pt": 0.13926658034324646, "Regression_eta": 0.023038089275360107, "Regression_sin_phi": 0.005813806317746639, "Regression_cos_phi": 0.00596633879467845, "Regression_energy": 0.1424562931060791, "Total": 8.345921516418457, "Total_unweighted": 2.864427498076111}, "train_group": {}, "valid": {"Classification_binary": 2.553987185160319, "Classification": 0.01798529550433159, "Regression_pt": 0.13526002566019693, "Regression_eta": 0.011365200082461039, "Regression_sin_phi": 0.006617686400810878, "Regression_cos_phi": 0.0070389943818251295, "Regression_energy": 0.14125712712605795, "Total": 7.972494761149089, "Total_unweighted": 2.8735115143160024, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01795784942805767, "diagnostic/loss/cld_pf/Classification_binary": 2.5505013465881348, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006752967834472656, "diagnostic/loss/cld_pf/Regression_energy": 0.14078640937805176, "diagnostic/loss/cld_pf/Regression_eta": 0.010560153983533382, "diagnostic/loss/cld_pf/Regression_pt": 0.13536155223846436, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006202369928359985, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07232212275266647, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.194402813911438, "diagnostic/regression/cld_pf/energy_residual_mean": 0.045202843844890594, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41876545548439026, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03513933718204498, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1876758188009262, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0623064786195755, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41170522570610046, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9394955635070801, "metrics/particle/class_1/pid_accuracy": 0.9858835935592651, "metrics/particle/class_2/efficiency": 0.46405377984046936, "metrics/particle/class_2/pid_accuracy": 0.7616007924079895, "metrics/particle/class_3/efficiency": 0.7855573296546936, "metrics/particle/class_3/pid_accuracy": 0.9843339323997498, "metrics/particle/class_4/efficiency": 0.8335015177726746, "metrics/particle/class_4/pid_accuracy": 0.3401937186717987, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6303030252456665, "metrics/particle/count/bias_mean": 0.16013072431087494, "metrics/particle/count/mae": 3.2124183177948, "metrics/particle/count/prediction_mean": 70.7254867553711, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05181577429175377, "metrics/particle/event/energy_response_mean": 0.9931584596633911, "metrics/particle/event/met_abs_error": 5.213191509246826, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04270130395889282, "metrics/particle/event/scalar_pt_response_mean": 1.002835750579834, "metrics/particle/event/vector_pt_closure": 0.04562544450163841, "metrics/particle/matched/delta_eta_abs_mean": 0.008428394794464111, "metrics/particle/matched/delta_phi_abs_mean": 0.00900090578943491, "metrics/particle/matched/delta_r_mean": 0.013887070119380951, "metrics/particle/matched/energy_relative_abs_mean": 0.0917675569653511, "metrics/particle/matched/pid_accuracy": 0.9672696590423584, "metrics/particle/matched/pt_relative_abs_mean": 0.08468877524137497, "metrics/particle/matching/duplicate_fraction": 0.03193789720535278, "metrics/particle/matching/efficiency": 0.835094690322876, "metrics/particle/matching/f1": 0.8341482877731323, "metrics/particle/matching/purity": 0.8332039713859558, "metrics/particle/matching/target_coverage_dr0p05": 0.821812629699707, "metrics/particle/matching/target_coverage_dr0p10": 0.8693928718566895, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7286620736122131, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8699578642845154, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01599165052175522, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08614122867584229, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08220475167036057, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014980445615947247, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13375480473041534, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005203291308134794, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3784966766834259, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05491292476654053, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5567375421524048, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8264.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2578140199184418, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8964183330535889, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9122833013534546, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8006582260131836, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12027943879365921, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.670531272888184, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012959400191903114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7021840214729309, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06757746636867523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.450326442718506, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9877.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8152313232421875, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.39580124616622925, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.3336615562438965, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.787353277206421, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11817941814661026, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.14843517541885376, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.468010902404785, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.9927469491958618, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.596962928771973, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024906082078814507, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0962923914194107, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09602714329957962, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01354303304105997, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14921772480010986, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0235294122248888, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0039873123168945, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3094632625579834, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.339252233505249, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1615.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7882152795791626, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5852874517440796, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 15637.2841796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 15637.1083984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26708075404167175, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 793503.3125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0892857164144516, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9307918548583984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.175790935754776, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.03870964050293, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2576.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5874863862991333, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.8425356149673462, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.72386360168457, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9915338158607483, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02888704091310501, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04570747911930084, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.576204299926758, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2788593769073486, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.007487297058105, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012012160383164883, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09455277770757675, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08615481108427048, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014238530769944191, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14082758128643036, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006010217475704849, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1143873780965805, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00744022848084569, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.512852668762207, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19966.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1441918909549713, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7654743790626526, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5755636096000671, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4642520546913147, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0889657586812973, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.628960609436035, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0030157885048538446, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2595192790031433, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004504385404288769, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7291812896728516, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22548.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8732886910438538, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.18696990609169006, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.704133033752441, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.679232120513916, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.08750340342521667, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.07798172533512115, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.950171947479248, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3039640188217163, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.221909046173096, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01381179690361023, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09231778234243393, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0855952724814415, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014406329020857811, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1393784135580063, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003116099862381816, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23565737903118134, "metrics/particle/momentum/all/accepted/pz_bias": -0.03692863509058952, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.204321265220642, "metrics/particle/momentum/all/accepted/samples": 29845.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2236539125442505, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8627621531486511, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1151.4993896484375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1151.3834228515625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11091111600399017, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 215268.984375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012171081267297268, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5810357928276062, "metrics/particle/momentum/all/assigned/pz_bias": -0.034909430891275406, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8942184448242188, "metrics/particle/momentum/all/assigned/samples": 35001.0, "metrics/particle/momentum/all/efficiency": 0.8348485231399536, "metrics/particle/momentum/all/event/energy_bias": -1.2597644329071045, "metrics/particle/momentum/all/event/p3_error_mean": 14.002384185791016, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.361211776733398, "metrics/particle/momentum/all/event/px_bias": -0.23456986248493195, "metrics/particle/momentum/all/event/py_bias": -0.024745948612689972, "metrics/particle/momentum/all/event/pz_abs_mean": 10.127123832702637, "metrics/particle/momentum/all/event/pz_bias": -1.9676423072814941, "metrics/particle/momentum/all/event/pz_residual_rms": 17.16764259338379, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008348796516656876, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.051884233951568604, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.039886508136987686, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005774465389549732, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0884881392121315, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018056160770356655, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14265955984592438, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0109584154561162, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8688069581985474, "metrics/particle/momentum/class_1/accepted/samples": 16061.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08357496559619904, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2640383839607239, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30861419439315796, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2524469196796417, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04434589669108391, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.570619583129883, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0055432370863854885, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3048555850982666, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.025630773976445198, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1911027431488037, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9368838667869568, "metrics/particle/momentum/class_1/event/energy_bias": 1.3232253789901733, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.953478813171387, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.5725321769714355, "metrics/particle/momentum/class_1/event/px_bias": 0.025797322392463684, "metrics/particle/momentum/class_1/event/py_bias": 0.09743764996528625, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.188014984130859, "metrics/particle/momentum/class_1/event/pz_bias": -0.32337161898612976, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.248924255371094, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03385484591126442, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19657555222511292, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1994239091873169, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04413088038563728, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23989102244377136, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03763837739825249, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.734574317932129, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.45147961378097534, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.236827373504639, "metrics/particle/momentum/class_2/accepted/samples": 1355.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8640086650848389, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7009572386741638, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 15250.5048828125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 15250.052734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35516849160194397, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 783677.6875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0802726224064827, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.98478364944458, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3631046712398529, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.688603401184082, "metrics/particle/momentum/class_2/assigned/samples": 2641.0, "metrics/particle/momentum/class_2/efficiency": 0.46885812282562256, "metrics/particle/momentum/class_2/event/energy_bias": -6.7165303230285645, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.741779327392578, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.959602355957031, "metrics/particle/momentum/class_2/event/px_bias": -0.08098119497299194, "metrics/particle/momentum/class_2/event/py_bias": 0.30747556686401367, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.098552703857422, "metrics/particle/momentum/class_2/event/pz_bias": -2.2529513835906982, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.913333892822266, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01902065984904766, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13529153168201447, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13430577516555786, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.031763434410095215, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17520098388195038, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008350033313035965, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18554948270320892, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.030118701979517937, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6973336935043335, "metrics/particle/momentum/class_3/accepted/samples": 11976.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2730291783809662, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5756484270095825, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4365272521972656, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3091776371002197, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1444028615951538, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.169032096862793, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007825790904462337, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.47388699650764465, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01041468232870102, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.488523244857788, "metrics/particle/momentum/class_3/assigned/samples": 14695.0, "metrics/particle/momentum/class_3/efficiency": 0.7884653210639954, "metrics/particle/momentum/class_3/event/energy_bias": 5.525552749633789, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.93892765045166, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.119377613067627, "metrics/particle/momentum/class_3/event/px_bias": -0.005465269088745117, "metrics/particle/momentum/class_3/event/py_bias": -0.5203171372413635, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.143373012542725, "metrics/particle/momentum/class_3/event/pz_bias": 0.11320260167121887, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.173759460449219, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011893034912645817, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11032125353813171, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11071491241455078, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05238969251513481, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1657625287771225, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010452961549162865, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5105106830596924, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13170257210731506, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8492929935455322, "metrics/particle/momentum/class_4/accepted/samples": 287.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2814555764198303, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8632152080535889, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8081190586090088, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7430095672607422, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15819208323955536, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.894519805908203, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6504446864128113, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07026312500238419, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1510207653045654, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8107344508171082, "metrics/particle/momentum/class_4/event/energy_bias": -1.3821804523468018, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7729980945587158, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3292555809020996, "metrics/particle/momentum/class_4/event/px_bias": -0.3127366900444031, "metrics/particle/momentum/class_4/event/py_bias": 0.09044775366783142, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9621187448501587, "metrics/particle/momentum/class_4/event/pz_bias": 0.34002262353897095, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5153777599334717, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006295529659837484, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021915212273597717, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021285096183419228, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009492776356637478, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05746058374643326, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13815799355506897, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.051359888166189194, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4611295163631439, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.012468858622014523, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.043279100209474564, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.043497685343027115, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.009718434885144234, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12962234020233154, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2044907510280609, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.009030192159116268, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8176192045211792, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.009831428527832031, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.894184410572052, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7240579128265381, "metrics/particle/momentum/class_5/event/px_bias": 0.1388159841299057, "metrics/particle/momentum/class_5/event/py_bias": 0.00021019577980041504, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4121979773044586, "metrics/particle/momentum/class_5/event/pz_bias": 0.15545551478862762, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7322889566421509, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4009315967559814, "step_valid_time": 7169.834437608719, "step_total_time": 7171.235369443893, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003817332771713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07616065381776482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2930, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2920, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616015753199869, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9647678630227198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9631821170282708, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03523213697728021, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9583196586806695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.961475139940731, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9598948060486522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616015753199869, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021621474696032, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0830892792162663, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2181, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2173, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9574187884108868, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574187884108868, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9574187884108868, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04258121158911326, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9539069359086918, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539069359086918, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539069359086918, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9574187884108868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031708859928115, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08374354589932254, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1325, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9343396226415094, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9368142262580401, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06566037735849056, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9313207547169812, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337873628452517, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9393019726858877} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_45000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..dbcbd5fdc275d96cf98e3c85c0991e0dc3e856f4 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131489200454718, "train": {"Classification_binary": 2.618093967437744, "Classification": 0.01724204793572426, "Regression_pt": 0.13642244040966034, "Regression_eta": 0.009865210391581059, "Regression_sin_phi": 0.006206084508448839, "Regression_cos_phi": 0.006181623321026564, "Regression_energy": 0.13751602172851562, "Total": 7.820043087005615, "Total_unweighted": 2.931527395732701}, "train_group": {}, "valid": {"Classification_binary": 2.5393282572428384, "Classification": 0.017985937496026356, "Regression_pt": 0.13309231400489807, "Regression_eta": 0.011389189710219702, "Regression_sin_phi": 0.006601577624678612, "Regression_cos_phi": 0.006996537248293559, "Regression_energy": 0.13898545503616333, "Total": 7.9040578206380205, "Total_unweighted": 2.854379268363118, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01795298233628273, "diagnostic/loss/cld_pf/Classification_binary": 2.5372724533081055, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006761505734175444, "diagnostic/loss/cld_pf/Regression_energy": 0.13902084529399872, "diagnostic/loss/cld_pf/Regression_eta": 0.010470181703567505, "diagnostic/loss/cld_pf/Regression_pt": 0.13371512293815613, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006220728624612093, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07642501592636108, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19426080584526062, "diagnostic/regression/cld_pf/energy_residual_mean": 0.049305740743875504, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4190683364868164, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03846246376633644, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18803633749485016, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06562960147857666, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4121038317680359, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9394189119338989, "metrics/particle/class_1/pid_accuracy": 0.9843319654464722, "metrics/particle/class_2/efficiency": 0.45596444606781006, "metrics/particle/class_2/pid_accuracy": 0.7558720707893372, "metrics/particle/class_3/efficiency": 0.7863264679908752, "metrics/particle/class_3/pid_accuracy": 0.9842094779014587, "metrics/particle/class_4/efficiency": 0.8365287780761719, "metrics/particle/class_4/pid_accuracy": 0.36911943554878235, "metrics/particle/class_5/efficiency": 0.9650485515594482, "metrics/particle/class_5/pid_accuracy": 0.6901408433914185, "metrics/particle/count/bias_mean": 0.17712418735027313, "metrics/particle/count/mae": 3.324836492538452, "metrics/particle/count/prediction_mean": 70.74248504638672, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.051341306418180466, "metrics/particle/event/energy_response_mean": 0.9965919256210327, "metrics/particle/event/met_abs_error": 5.129570484161377, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04153149202466011, "metrics/particle/event/scalar_pt_response_mean": 1.0039304494857788, "metrics/particle/event/vector_pt_closure": 0.04492184892296791, "metrics/particle/matched/delta_eta_abs_mean": 0.00814236793667078, "metrics/particle/matched/delta_phi_abs_mean": 0.009028278291225433, "metrics/particle/matched/delta_r_mean": 0.013698293827474117, "metrics/particle/matched/energy_relative_abs_mean": 0.09168170392513275, "metrics/particle/matched/pid_accuracy": 0.9668574929237366, "metrics/particle/matched/pt_relative_abs_mean": 0.0852634534239769, "metrics/particle/matching/duplicate_fraction": 0.0317084901034832, "metrics/particle/matching/efficiency": 0.8347705006599426, "metrics/particle/matching/f1": 0.8337242007255554, "metrics/particle/matching/purity": 0.8326804637908936, "metrics/particle/matching/target_coverage_dr0p05": 0.8215069770812988, "metrics/particle/matching/target_coverage_dr0p10": 0.8693835735321045, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.727874755859375, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8702820539474487, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015768663957715034, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08658374100923538, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08263867348432541, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015414571389555931, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1336536407470703, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0041172197088599205, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37691688537597656, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02968885749578476, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4753801822662354, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8258.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25416651368141174, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9071012139320374, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8912572264671326, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7772048711776733, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1183437928557396, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.980241298675537, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01336302887648344, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7234864234924316, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.028179118409752846, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.513662815093994, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9878.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8146394491195679, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6988133788108826, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.538940906524658, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8749966621398926, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.10828843712806702, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.17467236518859863, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.663074970245361, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5376197695732117, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.621891021728516, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025454634800553322, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09531041979789734, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09551309794187546, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017270145937800407, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14777106046676636, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020345252007246017, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9323431849479675, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.24784190952777863, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.079219341278076, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1622.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7557584643363953, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2748912572860718, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 20969.193359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 20969.021484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2605825364589691, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1063944.625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07844660431146622, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.881329298019409, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3014739751815796, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.087114334106445, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2575.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5900327563285828, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7691569924354553, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.121647357940674, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.018528938293457, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04203374683856964, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08366957306861877, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.96687126159668, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3743304014205933, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.267126083374023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011832878924906254, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09463909268379211, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08702129870653152, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017273487523198128, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14091423153877258, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005512402858585119, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11603306978940964, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0015026641776785254, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.497653067111969, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19955.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14398133754730225, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6296055912971497, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5187437534332275, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4078812599182129, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08823790401220322, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.67893123626709, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026187305338680744, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24465279281139374, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.006639453116804361, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2056117057800293, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22530.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8728076219558716, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.06230241060256958, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.736209869384766, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.634246826171875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.10730647295713425, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.26300376653671265, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0715932846069336, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.07977324724197388, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.979200839996338, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01366281695663929, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09244596213102341, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08626990020275116, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01675877906382084, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13932928442955017, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026143791619688272, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23262208700180054, "metrics/particle/momentum/all/accepted/pz_bias": -0.022696692496538162, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1329686641693115, "metrics/particle/momentum/all/accepted/samples": 29835.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22012503445148468, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7554585337638855, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1544.06884765625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1543.9527587890625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10942457616329193, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 288655.03125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01123402826488018, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5739372968673706, "metrics/particle/momentum/all/assigned/pz_bias": -0.025871479883790016, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9662067890167236, "metrics/particle/momentum/all/assigned/samples": 34983.0, "metrics/particle/momentum/all/efficiency": 0.8345687985420227, "metrics/particle/momentum/all/event/energy_bias": -0.008041217923164368, "metrics/particle/momentum/all/event/p3_error_mean": 14.345306396484375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.228660583496094, "metrics/particle/momentum/all/event/px_bias": 0.043015703558921814, "metrics/particle/momentum/all/event/py_bias": -0.3540065586566925, "metrics/particle/momentum/all/event/pz_abs_mean": 10.541866302490234, "metrics/particle/momentum/all/event/pz_bias": -1.832176923751831, "metrics/particle/momentum/all/event/pz_residual_rms": 19.60569190979004, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008798465132713318, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05213195085525513, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04114134982228279, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008710919879376888, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08911822736263275, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015559842577204108, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1497088521718979, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006905096583068371, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.890495240688324, "metrics/particle/momentum/class_1/accepted/samples": 16067.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08514629304409027, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24601484835147858, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2988739311695099, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2422156184911728, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.044000934809446335, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.578676223754883, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005077030975371599, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31376105546951294, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.016105974093079567, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.42523455619812, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9372338652610779, "metrics/particle/momentum/class_1/event/energy_bias": 1.4156084060668945, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.343986511230469, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.604437828063965, "metrics/particle/momentum/class_1/event/px_bias": 0.6471167802810669, "metrics/particle/momentum/class_1/event/py_bias": 0.03371991217136383, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.548158645629883, "metrics/particle/momentum/class_1/event/pz_bias": -0.8750649690628052, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.554006576538086, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033448804169893265, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19224150478839874, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19487264752388, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04401123896241188, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23363827168941498, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03148425742983818, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6552430391311646, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.40935930609703064, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8895246982574463, "metrics/particle/momentum/class_2/accepted/samples": 1334.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8589149117469788, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.676155149936676, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20513.521484375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20513.0625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3556230962276459, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1052361.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07788753509521484, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.014890193939209, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2389867752790451, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.285447120666504, "metrics/particle/momentum/class_2/assigned/samples": 2632.0, "metrics/particle/momentum/class_2/efficiency": 0.4615916907787323, "metrics/particle/momentum/class_2/event/energy_bias": -6.520116806030273, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.068143844604492, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.025877475738525, "metrics/particle/momentum/class_2/event/px_bias": -0.372305691242218, "metrics/particle/momentum/class_2/event/py_bias": 0.038484878838062286, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.465566635131836, "metrics/particle/momentum/class_2/event/pz_bias": -2.2748875617980957, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.71123218536377, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018115799874067307, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13566918671131134, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13474451005458832, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0331624299287796, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17558778822422028, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007515657343901694, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18048015236854553, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004771517124027014, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6251620054244995, "metrics/particle/momentum/class_3/accepted/samples": 11975.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26445233821868896, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3562817573547363, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.286197304725647, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1601451635360718, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1416122019290924, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.693357467651367, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006535947788506746, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4429042339324951, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.005275445524603128, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.114788293838501, "metrics/particle/momentum/class_3/assigned/samples": 14688.0, "metrics/particle/momentum/class_3/efficiency": 0.7883995175361633, "metrics/particle/momentum/class_3/event/energy_bias": 6.25205135345459, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.793060302734375, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.016963958740234, "metrics/particle/momentum/class_3/event/px_bias": -0.051519423723220825, "metrics/particle/momentum/class_3/event/py_bias": -0.5810098052024841, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.133580684661865, "metrics/particle/momentum/class_3/event/pz_bias": 0.7937502264976501, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.850726127624512, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012168706394731998, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.119206503033638, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11936816573143005, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06498657912015915, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1722414791584015, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006825938355177641, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4696561098098755, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.09735924750566483, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4945659637451172, "metrics/particle/momentum/class_4/accepted/samples": 293.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.25272396206855774, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3772351741790771, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5174353122711182, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4443838596343994, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14689265191555023, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.381775856018066, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6037987470626831, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15354427695274353, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9346868991851807, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8276836276054382, "metrics/particle/momentum/class_4/event/energy_bias": -1.4191404581069946, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.949235200881958, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5138400793075562, "metrics/particle/momentum/class_4/event/px_bias": -0.36196428537368774, "metrics/particle/momentum/class_4/event/py_bias": 0.11914808303117752, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9938638210296631, "metrics/particle/momentum/class_4/event/pz_bias": 0.40172407031059265, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.582063674926758, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00688177440315485, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02713794820010662, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026168329641222954, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.015600408427417278, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07188985496759415, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1684025228023529, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.05112282559275627, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.49073296785354614, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04140305519104004, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.14016292989253998, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.14061307907104492, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0941140353679657, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.212175726890564, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2724144458770752, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.13337722420692444, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2918320894241333, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.2635568380355835, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9269941449165344, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7372518181800842, "metrics/particle/momentum/class_5/event/px_bias": 0.1816883683204651, "metrics/particle/momentum/class_5/event/py_bias": 0.035650335252285004, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4340062141418457, "metrics/particle/momentum/class_5/event/pz_bias": 0.12230144441127777, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.799849033355713, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4636332988739014, "step_valid_time": 7310.972165107727, "step_total_time": 7312.4357986450195, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0062207443998874, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07357053830348648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2920, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625861503117821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9651201052977953, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9638514623726585, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03487989470220467, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9583196586806695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9608423823626193, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9595793624712455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625861503117821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048740755559813, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08110321280200616, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9591747146619842, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9604395604395605, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9598067208433999, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03956043956043956, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9560439560439561, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554140127388535, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9591747146619842, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0044048060546191, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08465263098135645, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1230, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.938403041825095, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9373338397265477, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061596958174904945, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9332321699544764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.935361216730038, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9342954804405621, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936267071320182} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_46000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..5b0c75625af495235cc53df16a46bec27a3f6ed6 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.1788556271314894, "train": {"Classification_binary": 2.4916858673095703, "Classification": 0.017895836383104324, "Regression_pt": 0.13382959365844727, "Regression_eta": 0.01853344216942787, "Regression_sin_phi": 0.007096848450601101, "Regression_cos_phi": 0.008049782365560532, "Regression_energy": 0.14389407634735107, "Total": 8.349296569824219, "Total_unweighted": 2.8209854466840625}, "train_group": {}, "valid": {"Classification_binary": 2.5436070760091147, "Classification": 0.01797015095750491, "Regression_pt": 0.13383878270785013, "Regression_eta": 0.01147435853878657, "Regression_sin_phi": 0.00655600739022096, "Regression_cos_phi": 0.0069780902316172915, "Regression_energy": 0.1398877501487732, "Total": 7.924765268961589, "Total_unweighted": 2.8603122159838676, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017922136932611465, "diagnostic/loss/cld_pf/Classification_binary": 2.5390496253967285, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00664735771715641, "diagnostic/loss/cld_pf/Regression_energy": 0.13944515585899353, "diagnostic/loss/cld_pf/Regression_eta": 0.010523642413318157, "diagnostic/loss/cld_pf/Regression_pt": 0.13402140140533447, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00606945576146245, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07339582592248917, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19540439546108246, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04627654328942299, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4174674153327942, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03612591326236725, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18888765573501587, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06329305469989777, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41173458099365234, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9395530819892883, "metrics/particle/class_1/pid_accuracy": 0.9850685596466064, "metrics/particle/class_2/efficiency": 0.46405377984046936, "metrics/particle/class_2/pid_accuracy": 0.7633194327354431, "metrics/particle/class_3/efficiency": 0.7850738167762756, "metrics/particle/class_3/pid_accuracy": 0.9834845066070557, "metrics/particle/class_4/efficiency": 0.8385469317436218, "metrics/particle/class_4/pid_accuracy": 0.3646209239959717, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6767676472663879, "metrics/particle/count/bias_mean": 0.24836601316928864, "metrics/particle/count/mae": 3.2601306438446045, "metrics/particle/count/prediction_mean": 70.81372833251953, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0512659065425396, "metrics/particle/event/energy_response_mean": 0.9889049530029297, "metrics/particle/event/met_abs_error": 4.907959461212158, "metrics/particle/event/scalar_pt_relative_abs_error": 0.041511885821819305, "metrics/particle/event/scalar_pt_response_mean": 0.9974094033241272, "metrics/particle/event/vector_pt_closure": 0.0443585030734539, "metrics/particle/matched/delta_eta_abs_mean": 0.00840592011809349, "metrics/particle/matched/delta_phi_abs_mean": 0.008847477845847607, "metrics/particle/matched/delta_r_mean": 0.013758745044469833, "metrics/particle/matched/energy_relative_abs_mean": 0.09333622455596924, "metrics/particle/matched/pid_accuracy": 0.967009425163269, "metrics/particle/matched/pt_relative_abs_mean": 0.08600465953350067, "metrics/particle/matching/duplicate_fraction": 0.03192579373717308, "metrics/particle/matching/efficiency": 0.8349650502204895, "metrics/particle/matching/f1": 0.8334982395172119, "metrics/particle/matching/purity": 0.832036554813385, "metrics/particle/matching/target_coverage_dr0p05": 0.8233872056007385, "metrics/particle/matching/target_coverage_dr0p10": 0.8703839182853699, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7252442836761475, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8701245784759521, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01569463312625885, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09117479622364044, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08685693889856339, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021348493173718452, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14020581543445587, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004592699930071831, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.39729973673820496, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.048683952540159225, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5611422061920166, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8274.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24733750522136688, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9815331697463989, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9877271056175232, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8801052570343018, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11676102131605148, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.31406307220459, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013445207849144936, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.734977662563324, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.042564067989587784, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.740353584289551, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9892.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.816217839717865, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6175495982170105, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.03576135635376, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0393970012664795, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.09523892402648926, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.015365894883871078, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.105146884918213, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.076025366783142, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.614757537841797, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024298906326293945, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10209260880947113, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.102483369410038, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.031087828800082207, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15572144091129303, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023617153987288475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.012179970741272, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.07457984983921051, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2907915115356445, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1609.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.769027829170227, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4336732625961304, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 20837.9921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 20837.82421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2667953670024872, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1060341.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08146718144416809, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8711953163146973, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.15109103918075562, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.526973724365234, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2590.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5853037238121033, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.4433567523956299, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.6690568923950195, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1758886575698853, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06556127965450287, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.005247030407190323, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.517001152038574, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.7729787826538086, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.39588737487793, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011917369440197945, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09489012509584427, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08609981834888458, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01047684345394373, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14094917476177216, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004012438585050404, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11572569608688354, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0063002826645970345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5113669037818909, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19938.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14165249466896057, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.560570240020752, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4514493942260742, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3361935317516327, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08849911391735077, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.815194129943848, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002664298517629504, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24823659658432007, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02360345423221588, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3970929384231567, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22520.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8720640540122986, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.221625566482544, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.45305061340332, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.380899429321289, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.8810008764266968, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.05749133974313736, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9643373489379883, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.432823121547699, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.1201863288879395, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013633441179990768, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09424789994955063, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08719386160373688, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01460531260818243, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14158084988594055, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028168070130050182, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24221837520599365, "metrics/particle/momentum/all/accepted/pz_bias": 0.013695957139134407, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.198055386543274, "metrics/particle/momentum/all/accepted/samples": 29821.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21794351935386658, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7441454529762268, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1542.4930419921875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1542.3758544921875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1096794456243515, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 288435.84375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011542197316884995, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5798835754394531, "metrics/particle/momentum/all/assigned/pz_bias": 0.03839550167322159, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.959439992904663, "metrics/particle/momentum/all/assigned/samples": 35002.0, "metrics/particle/momentum/all/efficiency": 0.8341771960258484, "metrics/particle/momentum/all/event/energy_bias": -2.0474326610565186, "metrics/particle/momentum/all/event/p3_error_mean": 14.188973426818848, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.9828009605407715, "metrics/particle/momentum/all/event/px_bias": -0.8513232469558716, "metrics/particle/momentum/all/event/py_bias": 0.03687841817736626, "metrics/particle/momentum/all/event/pz_abs_mean": 10.639650344848633, "metrics/particle/momentum/all/event/pz_bias": 3.281827211380005, "metrics/particle/momentum/all/event/pz_residual_rms": 19.96999740600586, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008618724532425404, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.052254367619752884, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03982226178050041, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004521307535469532, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08865650743246078, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001493744901381433, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.146382674574852, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0066222683526575565, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8723304271697998, "metrics/particle/momentum/class_1/accepted/samples": 16067.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08302885293960571, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23504745960235596, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2826940715312958, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2248551994562149, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04369895160198212, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.034788131713867, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005542590282857418, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31246107816696167, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0003793734358623624, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.493797779083252, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9372338652610779, "metrics/particle/momentum/class_1/event/energy_bias": 0.7101152539253235, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.44648551940918, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.619720458984375, "metrics/particle/momentum/class_1/event/px_bias": 0.09750908613204956, "metrics/particle/momentum/class_1/event/py_bias": 0.3624100089073181, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.749509811401367, "metrics/particle/momentum/class_1/event/pz_bias": -0.03569812700152397, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.52359962463379, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03332913666963577, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19896477460861206, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2018168419599533, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.047950442880392075, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24044790863990784, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0350484699010849, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7391622066497803, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17128074169158936, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.0344133377075195, "metrics/particle/momentum/class_2/accepted/samples": 1341.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8698415160179138, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7795802354812622, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20441.5234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20441.064453125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3526515066623688, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1050252.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07424242794513702, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9694273471832275, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04307014122605324, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.747208595275879, "metrics/particle/momentum/class_2/assigned/samples": 2640.0, "metrics/particle/momentum/class_2/efficiency": 0.46401384472846985, "metrics/particle/momentum/class_2/event/energy_bias": -7.44650411605835, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.499353408813477, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.731650352478027, "metrics/particle/momentum/class_2/event/px_bias": -0.4418935179710388, "metrics/particle/momentum/class_2/event/py_bias": 0.10386057943105698, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.942405700683594, "metrics/particle/momentum/class_2/event/pz_bias": -0.08046358823776245, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.589571952819824, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0183307696133852, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13960112631320953, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13839755952358246, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03435099869966507, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17963401973247528, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009201171342283487, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19912388920783997, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.058428309857845306, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8196952939033508, "metrics/particle/momentum/class_3/accepted/samples": 11955.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2604825496673584, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2624486684799194, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2696504592895508, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1425011157989502, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14310990273952484, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.874340057373047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007485539186745882, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4671512246131897, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07972913235425949, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3133068084716797, "metrics/particle/momentum/class_3/assigned/samples": 14695.0, "metrics/particle/momentum/class_3/efficiency": 0.7870827317237854, "metrics/particle/momentum/class_3/event/energy_bias": 5.802968502044678, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.350285530090332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.085296630859375, "metrics/particle/momentum/class_3/event/px_bias": -0.30868208408355713, "metrics/particle/momentum/class_3/event/py_bias": -0.472766637802124, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.74580192565918, "metrics/particle/momentum/class_3/event/pz_bias": 2.9746479988098145, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.953591346740723, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011066236533224583, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10734568536281586, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10792965441942215, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0512327179312706, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1600412279367447, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006825938355177641, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4557516872882843, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13767361640930176, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5181803703308105, "metrics/particle/momentum/class_4/accepted/samples": 293.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22506731748580933, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.91456937789917, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.3601953983306885, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.3075876235961914, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 35.68912124633789, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.568153440952301, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15705467760562897, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6957918405532837, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8276836276054382, "metrics/particle/momentum/class_4/event/energy_bias": -1.1917158365249634, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8062512874603271, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4089019298553467, "metrics/particle/momentum/class_4/event/px_bias": -0.3334757685661316, "metrics/particle/momentum/class_4/event/py_bias": 0.03331127390265465, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8850459456443787, "metrics/particle/momentum/class_4/event/pz_bias": 0.3218001425266266, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.182469367980957, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00608880165964365, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02303152158856392, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021705443039536476, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009244745597243309, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06403128802776337, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15142863988876343, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03433837369084358, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5054444670677185, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.008657979778945446, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.12910017371177673, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.12945711612701416, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1047082394361496, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1884926557540894, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2098187804222107, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.02026207372546196, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8368229269981384, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": 0.07770344614982605, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8124854564666748, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6576060056686401, "metrics/particle/momentum/class_5/event/px_bias": 0.1352190524339676, "metrics/particle/momentum/class_5/event/py_bias": 0.010063206776976585, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3809061348438263, "metrics/particle/momentum/class_5/event/pz_bias": 0.10154061019420624, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6576679944992065, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4074437618255615, "step_valid_time": 7452.472918987274, "step_total_time": 7453.880362987518, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994494264764143, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07876457900397194, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2923, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9657444005270093, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9639980272891665, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.034255599472990776, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9593042336724648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9627799736495388, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.961038961038961, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987910721105597, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08642071301630683, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2276, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2179, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2169, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.956540825285338, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9573813708260105, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9569609134826527, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04261862917398945, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9521510096575944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.95298769771529, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9525691699604744, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.956540825285338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001476957547347, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07852465947223142, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9391634980988594, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9380934295480441, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060836501901140684, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9376425855513308, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9365742499050512, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9370257966616085} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_47000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..719003515604273e567ec507625bc1c91db2ff81 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226222053808261, "train": {"Classification_binary": 2.5705339908599854, "Classification": 0.020227225497364998, "Regression_pt": 0.11852442473173141, "Regression_eta": 0.010291061364114285, "Regression_sin_phi": 0.006417705211788416, "Regression_cos_phi": 0.006286952644586563, "Regression_energy": 0.12090577185153961, "Total": 7.536452293395996, "Total_unweighted": 2.8531871321611106}, "train_group": {}, "valid": {"Classification_binary": 2.535804589589437, "Classification": 0.017958026379346848, "Regression_pt": 0.1335595746835073, "Regression_eta": 0.01137838140130043, "Regression_sin_phi": 0.006564738228917122, "Regression_cos_phi": 0.007011224826176961, "Regression_energy": 0.13942291339238486, "Total": 7.906803131103516, "Total_unweighted": 2.8516994485010705, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017883211374282837, "diagnostic/loss/cld_pf/Classification_binary": 2.5335893630981445, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006717721465975046, "diagnostic/loss/cld_pf/Regression_energy": 0.14051903784275055, "diagnostic/loss/cld_pf/Regression_eta": 0.010717526078224182, "diagnostic/loss/cld_pf/Regression_pt": 0.13547880947589874, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006107825320214033, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08735354989767075, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19754832983016968, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06023426353931427, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42344987392425537, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04882540926337242, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1910393238067627, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07599255442619324, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41594359278678894, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9395530819892883, "metrics/particle/class_1/pid_accuracy": 0.9853745102882385, "metrics/particle/class_2/efficiency": 0.4650791883468628, "metrics/particle/class_2/pid_accuracy": 0.7574717998504639, "metrics/particle/class_3/efficiency": 0.7779096364974976, "metrics/particle/class_3/pid_accuracy": 0.984688401222229, "metrics/particle/class_4/efficiency": 0.8193743824958801, "metrics/particle/class_4/pid_accuracy": 0.378078818321228, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.6545454263687134, "metrics/particle/count/bias_mean": 0.012418300844728947, "metrics/particle/count/mae": 3.2542483806610107, "metrics/particle/count/prediction_mean": 70.57777404785156, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.052453916519880295, "metrics/particle/event/energy_response_mean": 0.9971739649772644, "metrics/particle/event/met_abs_error": 4.94418478012085, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04305075854063034, "metrics/particle/event/scalar_pt_response_mean": 1.0087121725082397, "metrics/particle/event/vector_pt_closure": 0.045409806072711945, "metrics/particle/matched/delta_eta_abs_mean": 0.007953976280987263, "metrics/particle/matched/delta_phi_abs_mean": 0.008892328478395939, "metrics/particle/matched/delta_r_mean": 0.013439449481666088, "metrics/particle/matched/energy_relative_abs_mean": 0.0928189679980278, "metrics/particle/matched/pid_accuracy": 0.9674316048622131, "metrics/particle/matched/pt_relative_abs_mean": 0.08613772690296173, "metrics/particle/matching/duplicate_fraction": 0.03357904776930809, "metrics/particle/matching/efficiency": 0.831852912902832, "metrics/particle/matching/f1": 0.8317797183990479, "metrics/particle/matching/purity": 0.8317065238952637, "metrics/particle/matching/target_coverage_dr0p05": 0.8230260014533997, "metrics/particle/matching/target_coverage_dr0p10": 0.8698652386665344, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.722048819065094, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8669754266738892, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015436551533639431, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08931215107440948, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08571573346853256, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.027799513190984726, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13851433992385864, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0048555475659668446, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.39068928360939026, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008532414212822914, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5658142566680908, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8238.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2483035922050476, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9354298114776611, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8638710379600525, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7620286345481873, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12233127653598785, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.726970672607422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01416573952883482, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7351087927818298, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02555687353014946, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7795395851135254, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9883.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8126664757728577, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8982290029525757, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.331934452056885, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.870985269546509, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.035953305661678314, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2403157353401184, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.431321620941162, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.848305881023407, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.523128509521484, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02482871524989605, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09650217741727829, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09655820578336716, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02458372339606285, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14832936227321625, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023602483794093132, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.002964973449707, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2392934113740921, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2375407218933105, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1610.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.752192497253418, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4465222358703613, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 19369.134765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 19368.966796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2675507068634033, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 980626.1875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08268330991268158, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8682174682617188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.18736423552036285, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.493701934814453, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2564.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5856674909591675, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2304468154907227, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.1431121826171875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0333058834075928, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.022141460329294205, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08445988595485687, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.990043640136719, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2799967527389526, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.139659881591797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0114810261875391, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09488541632890701, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08680910617113113, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021383516490459442, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1418004184961319, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005036007496528327, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11391715705394745, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0030846167355775833, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.48782360553741455, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19857.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13907639682292938, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6718379855155945, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5736539363861084, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.471024751663208, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09361399710178375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.250384330749512, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027977616991847754, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24696780741214752, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0006754688802175224, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2730385065078735, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22518.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8685212135314941, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5763163566589355, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.076904296875, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.088549613952637, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2946661710739136, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.08335255086421967, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.7839529514312744, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.23715758323669434, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.988226413726807, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013301441445946693, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09342742711305618, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08703427761793137, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023336296901106834, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14126063883304596, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029624642338603735, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.238859623670578, "metrics/particle/momentum/all/accepted/pz_bias": -0.017397867515683174, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1862224340438843, "metrics/particle/momentum/all/accepted/samples": 29705.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21491000056266785, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8031512498855591, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1420.9615478515625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1420.8543701171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11448591202497482, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 265549.65625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01186901144683361, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5771602988243103, "metrics/particle/momentum/all/assigned/pz_bias": -0.02052825130522251, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9185147285461426, "metrics/particle/momentum/all/assigned/samples": 34965.0, "metrics/particle/momentum/all/efficiency": 0.8309323191642761, "metrics/particle/momentum/all/event/energy_bias": 0.24409858882427216, "metrics/particle/momentum/all/event/p3_error_mean": 13.721587181091309, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.782516956329346, "metrics/particle/momentum/all/event/px_bias": -0.30847805738449097, "metrics/particle/momentum/all/event/py_bias": -0.07250331342220306, "metrics/particle/momentum/all/event/pz_abs_mean": 10.260656356811523, "metrics/particle/momentum/all/event/pz_bias": -1.8911449909210205, "metrics/particle/momentum/all/event/pz_residual_rms": 18.85089874267578, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008313220925629139, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05145303159952164, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04017738997936249, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008338919840753078, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08876345306634903, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014324863441288471, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14619068801403046, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01126815564930439, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8964649438858032, "metrics/particle/momentum/class_1/accepted/samples": 16056.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08058734238147736, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2649880051612854, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30257588624954224, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24940833449363708, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.044284965842962265, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.554630279541016, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005426221061497927, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31232863664627075, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.025623749941587448, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4347288608551025, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9365922212600708, "metrics/particle/momentum/class_1/event/energy_bias": 1.6254699230194092, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.099377632141113, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.493953704833984, "metrics/particle/momentum/class_1/event/px_bias": 0.1720530390739441, "metrics/particle/momentum/class_1/event/py_bias": -0.12382585555315018, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.325068473815918, "metrics/particle/momentum/class_1/event/pz_bias": -1.1326720714569092, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.089622497558594, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033304762095212936, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19554439187049866, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19780707359313965, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04464801028370857, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23640228807926178, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04065040498971939, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7437156438827515, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.28872594237327576, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.1212158203125, "metrics/particle/momentum/class_2/accepted/samples": 1353.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.834058940410614, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8484188318252563, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 18931.853515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 18931.416015625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3545558452606201, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 969534.6875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08311094343662262, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.010645866394043, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1750348061323166, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.7503156661987305, "metrics/particle/momentum/class_2/assigned/samples": 2623.0, "metrics/particle/momentum/class_2/efficiency": 0.4681660830974579, "metrics/particle/momentum/class_2/event/energy_bias": -7.292291164398193, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.41560173034668, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.797308921813965, "metrics/particle/momentum/class_2/event/px_bias": -0.293435662984848, "metrics/particle/momentum/class_2/event/py_bias": 0.18984568119049072, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.843624114990234, "metrics/particle/momentum/class_2/event/pz_bias": -1.9588851928710938, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.555992126464844, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017913954332470894, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13935473561286926, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13834474980831146, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05081954970955849, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.179843008518219, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006754474597983062, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18909169733524323, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0033041539136320353, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6698989272117615, "metrics/particle/momentum/class_3/accepted/samples": 11844.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2618488371372223, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3934067487716675, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3507388830184937, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2380726337432861, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1529708355665207, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.825241088867188, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006745707243680954, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.45599013566970825, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.010667167603969574, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.268726348876953, "metrics/particle/momentum/class_3/assigned/samples": 14676.0, "metrics/particle/momentum/class_3/efficiency": 0.7797748446464539, "metrics/particle/momentum/class_3/event/energy_bias": 7.044652938842773, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.967316627502441, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.071869373321533, "metrics/particle/momentum/class_3/event/px_bias": -0.18550077080726624, "metrics/particle/momentum/class_3/event/py_bias": -0.260531485080719, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.329984188079834, "metrics/particle/momentum/class_3/event/pz_bias": 0.7317763566970825, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.416773796081543, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011872510425746441, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10703420639038086, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1076846793293953, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05706227570772171, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15782734751701355, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00699300691485405, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.43942564725875854, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.03975947946310043, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4068470001220703, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28350046277046204, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4247567653656006, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8236349821090698, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7624561786651611, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17796610295772552, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.614631652832031, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5707597136497498, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08597874641418457, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6975464820861816, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.2660160064697266, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.67523193359375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2772022485733032, "metrics/particle/momentum/class_4/event/px_bias": -0.1462366282939911, "metrics/particle/momentum/class_4/event/py_bias": 0.07869970798492432, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8761627078056335, "metrics/particle/momentum/class_4/event/pz_bias": 0.335640549659729, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.0940628051757812, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006099163554608822, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02066972851753235, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01975397951900959, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009019147604703903, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.050993118435144424, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14193961024284363, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.025653310120105743, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.45304450392723083, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.012440836988389492, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.041331157088279724, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.041478805243968964, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01108656357973814, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12383025139570236, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20997275412082672, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0374358706176281, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9096301198005676, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.13228295743465424, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8588931560516357, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6999669075012207, "metrics/particle/momentum/class_5/event/px_bias": 0.1446419507265091, "metrics/particle/momentum/class_5/event/py_bias": 0.043308619409799576, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3945162296295166, "metrics/particle/momentum/class_5/event/pz_bias": 0.1329953521490097, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.675410270690918, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4695734977722168, "step_valid_time": 7593.852725028992, "step_total_time": 7595.322299003601, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0084583356277017, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0767415824126092, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2937, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2930, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9638989169675091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9651659546500164, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9645320197044335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03483404534998357, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9616015753199869, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9628655931646402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9622331691297209, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9638989169675091, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0074608357655186, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08373532935781225, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2281, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9535291538798772, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9541566132923887, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046470846120122755, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9491451117930733, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497696863347225, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547848990342406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0091079556162637, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08291952923946921, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9391634980988594, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9380934295480441, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060836501901140684, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9361216730038023, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9350550702620585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9370257966616085} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_48000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..afbbf2c685d56a2541a1a5b02aa975c1a02ada49 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.2735884804850324, "train": {"Classification_binary": 2.463191509246826, "Classification": 0.018731852993369102, "Regression_pt": 0.12004220485687256, "Regression_eta": 0.00989820808172226, "Regression_sin_phi": 0.006614748388528824, "Regression_cos_phi": 0.006562863942235708, "Regression_energy": 0.12350142002105713, "Total": 7.438263893127441, "Total_unweighted": 2.7485428075306118}, "train_group": {}, "valid": {"Classification_binary": 2.522416114807129, "Classification": 0.01777926708261172, "Regression_pt": 0.13117722670237222, "Regression_eta": 0.011315128455559412, "Regression_sin_phi": 0.006606507425506909, "Regression_cos_phi": 0.006990536426504453, "Regression_energy": 0.13697630167007446, "Total": 7.831419626871745, "Total_unweighted": 2.833261082569758, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01771850883960724, "diagnostic/loss/cld_pf/Classification_binary": 2.525336742401123, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006645850837230682, "diagnostic/loss/cld_pf/Regression_energy": 0.13694146275520325, "diagnostic/loss/cld_pf/Regression_eta": 0.010457184165716171, "diagnostic/loss/cld_pf/Regression_pt": 0.13192328810691833, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006132541224360466, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07530006021261215, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19373595714569092, "diagnostic/regression/cld_pf/energy_residual_mean": 0.048180777579545975, "diagnostic/regression/cld_pf/energy_residual_rms": 0.416881799697876, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03876625746488571, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18746648728847504, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06593339890241623, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41060879826545715, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9413545727729797, "metrics/particle/class_1/pid_accuracy": 0.9862168431282043, "metrics/particle/class_2/efficiency": 0.4620029628276825, "metrics/particle/class_2/pid_accuracy": 0.7620221972465515, "metrics/particle/class_3/efficiency": 0.7867439985275269, "metrics/particle/class_3/pid_accuracy": 0.9848882555961609, "metrics/particle/class_4/efficiency": 0.8435923457145691, "metrics/particle/class_4/pid_accuracy": 0.3516746461391449, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.6512096524238586, "metrics/particle/count/bias_mean": 0.2535947859287262, "metrics/particle/count/mae": 3.181699275970459, "metrics/particle/count/prediction_mean": 70.81895446777344, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.049769412726163864, "metrics/particle/event/energy_response_mean": 0.9904643297195435, "metrics/particle/event/met_abs_error": 5.029881954193115, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040536221116781235, "metrics/particle/event/scalar_pt_response_mean": 1.001213788986206, "metrics/particle/event/vector_pt_closure": 0.04445526376366615, "metrics/particle/matched/delta_eta_abs_mean": 0.007988367229700089, "metrics/particle/matched/delta_phi_abs_mean": 0.00883493758738041, "metrics/particle/matched/delta_r_mean": 0.013409505598247051, "metrics/particle/matched/energy_relative_abs_mean": 0.09167451411485672, "metrics/particle/matched/pid_accuracy": 0.9679087400436401, "metrics/particle/matched/pt_relative_abs_mean": 0.08494207262992859, "metrics/particle/matching/duplicate_fraction": 0.03181268647313118, "metrics/particle/matching/efficiency": 0.8364284634590149, "metrics/particle/matching/f1": 0.8349282145500183, "metrics/particle/matching/purity": 0.8334333300590515, "metrics/particle/matching/target_coverage_dr0p05": 0.824582040309906, "metrics/particle/matching/target_coverage_dr0p10": 0.8720604181289673, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7289584875106812, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8704487681388855, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015192050486803055, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08561981469392776, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08258907496929169, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01883498765528202, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13428322970867157, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004604386165738106, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37822192907333374, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.005483223590999842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5285249948501587, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8253.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.251568078994751, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8593574166297913, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9228106737136841, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8140553832054138, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12156148999929428, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.954843521118164, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013754045590758324, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7261834740638733, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.033028438687324524, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6284728050231934, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9888.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8141462206840515, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.029707670211791992, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.3072075843811035, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8548619747161865, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.07369555532932281, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2166612446308136, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.408483505249023, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.12949562072753906, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.284235000610352, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02393006533384323, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0951884537935257, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09612530469894409, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.020771006122231483, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14918968081474304, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022291021421551704, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9831382632255554, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.23054753243923187, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4340012073516846, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1615.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7728592157363892, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4282808303833008, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 20448.7265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 20448.556640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26903748512268066, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1039927.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08078856021165848, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.893763780593872, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.28097397089004517, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.891207695007324, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2587.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5874863862991333, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.9987413883209229, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.011437892913818, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9723705649375916, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03361145034432411, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09877197444438934, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.888598442077637, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3067857027053833, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.361202239990234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0116496616974473, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0946318656206131, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08630044013261795, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015466718934476376, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1412276327610016, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005002751713618636, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11389254778623581, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.003233567578718066, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.507682740688324, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19989.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13816411793231964, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6644579172134399, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5048947930335999, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.39412879943847656, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08919937908649445, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.932312488555908, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003193612676113844, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.25277453660964966, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012228486128151417, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4318815469741821, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22545.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8742946982383728, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.3900315463542938, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.737149238586426, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.629366874694824, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.24468201398849487, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.21218569576740265, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.089686393737793, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.22062933444976807, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.355494499206543, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013293102383613586, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09217087924480438, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0858059972524643, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01668468303978443, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13979078829288483, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028134104795753956, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2339763343334198, "metrics/particle/momentum/all/accepted/pz_bias": -0.01615108922123909, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2067465782165527, "metrics/particle/momentum/all/accepted/samples": 29857.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2170702964067459, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7759134769439697, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1511.175537109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1511.0609130859375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1116219311952591, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 282646.21875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011907481588423252, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5815382599830627, "metrics/particle/momentum/all/assigned/pz_bias": -0.0222094114869833, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0183627605438232, "metrics/particle/momentum/all/assigned/samples": 35020.0, "metrics/particle/momentum/all/efficiency": 0.8351842164993286, "metrics/particle/momentum/all/event/energy_bias": -2.418480634689331, "metrics/particle/momentum/all/event/p3_error_mean": 14.074183464050293, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.1313323974609375, "metrics/particle/momentum/all/event/px_bias": -0.20459792017936707, "metrics/particle/momentum/all/event/py_bias": -0.3300749659538269, "metrics/particle/momentum/all/event/pz_abs_mean": 10.375944137573242, "metrics/particle/momentum/all/event/pz_bias": -1.6569106578826904, "metrics/particle/momentum/all/event/pz_residual_rms": 18.831418991088867, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008189498446881771, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05194864049553871, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.040199849754571915, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006535703781992197, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08939220011234283, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015531808603554964, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1445583552122116, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.009488454088568687, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8921912312507629, "metrics/particle/momentum/class_1/accepted/samples": 16096.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08115287125110626, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.22863228619098663, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.28876781463623047, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23187202215194702, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.043409768491983414, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.454933166503906, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005601260345429182, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31131479144096375, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01788453198969364, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.45871639251709, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9389255046844482, "metrics/particle/momentum/class_1/event/energy_bias": 0.6256330013275146, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.961423873901367, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.335251808166504, "metrics/particle/momentum/class_1/event/px_bias": 0.16499385237693787, "metrics/particle/momentum/class_1/event/py_bias": -0.010853737592697144, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.339076519012451, "metrics/particle/momentum/class_1/event/pz_bias": -0.6110316514968872, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.05845069885254, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033371295779943466, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1949012130498886, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19732268154621124, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.042608097195625305, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23603838682174683, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0359012708067894, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7346960306167603, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2658301889896393, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.099868297576904, "metrics/particle/momentum/class_2/accepted/samples": 1337.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8542364835739136, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7052445411682129, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20021.056640625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20020.595703125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3641180992126465, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1029046.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07721423357725143, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.973057985305786, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.23208341002464294, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.831860065460205, "metrics/particle/momentum/class_2/assigned/samples": 2642.0, "metrics/particle/momentum/class_2/efficiency": 0.4626297652721405, "metrics/particle/momentum/class_2/event/energy_bias": -7.401332855224609, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.737311363220215, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.890918731689453, "metrics/particle/momentum/class_2/event/px_bias": -0.10009713470935822, "metrics/particle/momentum/class_2/event/py_bias": 0.23364898562431335, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.153421401977539, "metrics/particle/momentum/class_2/event/pz_bias": -1.8694273233413696, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.872636795043945, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01806563325226307, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13519147038459778, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13483826816082, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.036178018897771835, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17612718045711517, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007521939114667475, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18161281943321228, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0013846812071278691, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7849873900413513, "metrics/particle/momentum/class_3/accepted/samples": 11965.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26243528723716736, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3835135698318481, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3545893430709839, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.232475996017456, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14607124030590057, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.019975662231445, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007544861175119877, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.46833130717277527, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0028851786628365517, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4677975177764893, "metrics/particle/momentum/class_3/assigned/samples": 14712.0, "metrics/particle/momentum/class_3/efficiency": 0.7877411246299744, "metrics/particle/momentum/class_3/event/energy_bias": 5.763960361480713, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.919245719909668, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.9694414138793945, "metrics/particle/momentum/class_3/event/px_bias": -0.20186738669872284, "metrics/particle/momentum/class_3/event/py_bias": -0.6870954036712646, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.332897663116455, "metrics/particle/momentum/class_3/event/pz_bias": 0.40760332345962524, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.660346984863281, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011089292354881763, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11467441916465759, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11547844111919403, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.055219441652297974, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1699080616235733, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006825938355177641, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.48877212405204773, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12849919497966766, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6401768922805786, "metrics/particle/momentum/class_4/accepted/samples": 293.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24195867776870728, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.904221773147583, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4024248123168945, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3367083072662354, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1355932205915451, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.55917739868164, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6840049624443054, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2316940426826477, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.520031690597534, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8276836276054382, "metrics/particle/momentum/class_4/event/energy_bias": -1.3605130910873413, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7567849159240723, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.355457067489624, "metrics/particle/momentum/class_4/event/px_bias": -0.19434970617294312, "metrics/particle/momentum/class_4/event/py_bias": 0.15907776355743408, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.893526554107666, "metrics/particle/momentum/class_4/event/pz_bias": 0.36668670177459717, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.163609743118286, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006337855011224747, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024290254339575768, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023236572742462158, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005262680817395449, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06562212854623795, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14171597361564636, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.029129421338438988, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4372539222240448, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04294050112366676, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04832993075251579, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04837960749864578, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.003914432600140572, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15046748518943787, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.24740344285964966, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1008514016866684, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2765907049179077, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.0462280809879303, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.6987115144729614, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5713161826133728, "metrics/particle/momentum/class_5/event/px_bias": 0.12672246992588043, "metrics/particle/momentum/class_5/event/py_bias": -0.024852532893419266, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3136535584926605, "metrics/particle/momentum/class_5/event/pz_bias": 0.04925837367773056, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4155797958374023, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.462174415588379, "step_valid_time": 7735.072160959244, "step_total_time": 7736.534335851669, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004019542412099, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07178190549639263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3038, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2923, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9619297669839186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9647794601711652, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9633525061626952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03522053982883476, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9593042336724648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9621461487820935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9607230895645029, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9619297669839186, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0022959894228023, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08251225565846032, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2177, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2170, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04433713784021071, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9525899912203687, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9525899912203687, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9525899912203687, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9556628621597892, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026917853814352, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0842189619260415, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1320, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9340909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9347990902198635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0659090909090909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9325757575757576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9332827899924185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9355083459787557} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_49000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..dd17aae328871830be00ebb6a2e7ecf8bb62787b --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.320954907161804, "train": {"Classification_binary": 2.439523220062256, "Classification": 0.017997700721025467, "Regression_pt": 0.12218721956014633, "Regression_eta": 0.010204946622252464, "Regression_sin_phi": 0.006129125133156776, "Regression_cos_phi": 0.006892838049679995, "Regression_energy": 0.12669721245765686, "Total": 7.436517238616943, "Total_unweighted": 2.7296322626061738}, "train_group": {}, "valid": {"Classification_binary": 2.5357418060302734, "Classification": 0.017841463287671406, "Regression_pt": 0.13288154204686484, "Regression_eta": 0.011335791399081549, "Regression_sin_phi": 0.0065734342982371645, "Regression_cos_phi": 0.007003797839085261, "Regression_energy": 0.138707826534907, "Total": 7.88491948445638, "Total_unweighted": 2.8500856614361205, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017748476937413216, "diagnostic/loss/cld_pf/Classification_binary": 2.5385773181915283, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006702302489429712, "diagnostic/loss/cld_pf/Regression_energy": 0.1385042816400528, "diagnostic/loss/cld_pf/Regression_eta": 0.010590440593659878, "diagnostic/loss/cld_pf/Regression_pt": 0.13338418304920197, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006157112307846546, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07611901313066483, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19397346675395966, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04899973049759865, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41959404945373535, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04058842360973358, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18766461312770844, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0677555650472641, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41392961144447327, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9393805861473083, "metrics/particle/class_1/pid_accuracy": 0.985779881477356, "metrics/particle/class_2/efficiency": 0.4700922966003418, "metrics/particle/class_2/pid_accuracy": 0.7656325697898865, "metrics/particle/class_3/efficiency": 0.787007749080658, "metrics/particle/class_3/pid_accuracy": 0.9842790365219116, "metrics/particle/class_4/efficiency": 0.8324924111366272, "metrics/particle/class_4/pid_accuracy": 0.3527272641658783, "metrics/particle/class_5/efficiency": 0.9669902920722961, "metrics/particle/class_5/pid_accuracy": 0.6244980096817017, "metrics/particle/count/bias_mean": 0.1522875875234604, "metrics/particle/count/mae": 3.194117546081543, "metrics/particle/count/prediction_mean": 70.71764373779297, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.049974311143159866, "metrics/particle/event/energy_response_mean": 0.9962167143821716, "metrics/particle/event/met_abs_error": 5.047292232513428, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04056636989116669, "metrics/particle/event/scalar_pt_response_mean": 1.008570671081543, "metrics/particle/event/vector_pt_closure": 0.04458492621779442, "metrics/particle/matched/delta_eta_abs_mean": 0.008096381090581417, "metrics/particle/matched/delta_phi_abs_mean": 0.008885478600859642, "metrics/particle/matched/delta_r_mean": 0.013532273471355438, "metrics/particle/matched/energy_relative_abs_mean": 0.09147182106971741, "metrics/particle/matched/pid_accuracy": 0.9673445820808411, "metrics/particle/matched/pt_relative_abs_mean": 0.08455368131399155, "metrics/particle/matching/duplicate_fraction": 0.03296733647584915, "metrics/particle/matching/efficiency": 0.836159884929657, "metrics/particle/matching/f1": 0.8352586030960083, "metrics/particle/matching/purity": 0.8343592286109924, "metrics/particle/matching/target_coverage_dr0p05": 0.8222386837005615, "metrics/particle/matching/target_coverage_dr0p10": 0.8703468441963196, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7300236225128174, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8709582090377808, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015704089775681496, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0865444764494896, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08345117419958115, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015949087217450142, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1356324851512909, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004703895654529333, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37847772240638733, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.018259692937135696, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5425963401794434, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8291.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25365152955055237, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0375707149505615, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1393191814422607, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0294768810272217, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11553623527288437, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.246694564819336, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013848175294697285, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.726779580116272, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.018133608624339104, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.622360944747925, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9893.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8178948163986206, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.2334643602371216, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.1741251945495605, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.105905532836914, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2592623829841614, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.0942581444978714, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.255980968475342, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5101135969161987, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.481597900390625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025068730115890503, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09293457120656967, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09441754966974258, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018834013491868973, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14531338214874268, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02675793319940567, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9587225914001465, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15601252019405365, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0672409534454346, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1607.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7735031843185425, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9113261699676514, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21296.02734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21295.86328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2651869058609009, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1079005.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08489096909761429, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.765768051147461, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.16707614064216614, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.112295150756836, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2568.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5845761895179749, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.3122940063476562, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.375535488128662, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1156359910964966, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0651932954788208, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09414860606193542, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.203892707824707, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7286195755004883, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.31755256652832, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011599735356867313, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09423358738422394, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08571287244558334, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016126753762364388, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14052045345306396, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005512955249287188, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11522605270147324, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004489701706916094, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5205680131912231, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19953.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14314426481723785, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6506054997444153, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5318269729614258, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42352384328842163, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09108252078294754, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.723369121551514, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029281277675181627, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2552134692668915, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013165759854018688, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.567243218421936, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22540.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8727201223373413, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.2766203880310059, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.371726036071777, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.313534736633301, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.24794891476631165, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.22854343056678772, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9095568656921387, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.17771011590957642, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.999336242675781, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013464793562889099, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09202803671360016, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08555330336093903, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016223149374127388, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1394447535276413, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003115473547950387, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23375201225280762, "metrics/particle/momentum/all/accepted/pz_bias": -0.0003262201207689941, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1612625122070312, "metrics/particle/momentum/all/accepted/samples": 29851.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2206280529499054, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.852479100227356, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1563.1397705078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1563.02685546875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11076826602220535, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 292267.8125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012028227560222149, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5726989507675171, "metrics/particle/momentum/all/assigned/pz_bias": 0.0013456889428198338, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.890308141708374, "metrics/particle/momentum/all/assigned/samples": 35001.0, "metrics/particle/momentum/all/efficiency": 0.8350163698196411, "metrics/particle/momentum/all/event/energy_bias": 0.19779065251350403, "metrics/particle/momentum/all/event/p3_error_mean": 14.028839111328125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.099541664123535, "metrics/particle/momentum/all/event/px_bias": 0.07650676369667053, "metrics/particle/momentum/all/event/py_bias": 0.22843390703201294, "metrics/particle/momentum/all/event/pz_abs_mean": 10.452324867248535, "metrics/particle/momentum/all/event/pz_bias": -0.040795858949422836, "metrics/particle/momentum/all/event/pz_residual_rms": 19.605825424194336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008200589567422867, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.051585275679826736, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0393262654542923, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0065191504545509815, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08803761750459671, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016822429606691003, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14360804855823517, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.009008165448904037, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8880269527435303, "metrics/particle/momentum/class_1/accepted/samples": 16050.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08258061856031418, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2569192945957184, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3103383481502533, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2551857829093933, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.044871047139167786, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.351635456085205, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005718287080526352, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.30979034304618835, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02158709615468979, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3966894149780273, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9362422227859497, "metrics/particle/momentum/class_1/event/energy_bias": 1.7893208265304565, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.056753158569336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.39658260345459, "metrics/particle/momentum/class_1/event/px_bias": 0.13248004019260406, "metrics/particle/momentum/class_1/event/py_bias": 0.06899601221084595, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.454671859741211, "metrics/particle/momentum/class_1/event/pz_bias": -0.6907622814178467, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.1496524810791, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03356299176812172, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1953393518924713, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1987856775522232, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04382937029004097, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23915739357471466, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03851744160056114, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7223939895629883, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.18370947241783142, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9883058071136475, "metrics/particle/momentum/class_2/accepted/samples": 1376.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8216416835784912, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7102571129798889, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 20767.845703125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 20767.40625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34713253378868103, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1065603.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07368021458387375, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8322794437408447, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.11752765625715256, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.472994327545166, "metrics/particle/momentum/class_2/assigned/samples": 2633.0, "metrics/particle/momentum/class_2/efficiency": 0.4761245548725128, "metrics/particle/momentum/class_2/event/energy_bias": -6.02186393737793, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.619013786315918, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.838225841522217, "metrics/particle/momentum/class_2/event/px_bias": -0.1247202455997467, "metrics/particle/momentum/class_2/event/py_bias": 0.5631227493286133, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.093846321105957, "metrics/particle/momentum/class_2/event/pz_bias": -1.2267279624938965, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.878995895385742, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01836109533905983, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13497191667556763, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13484667241573334, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03527989611029625, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17564842104911804, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008356313337571919, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17816682159900665, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.029211347922682762, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6375343203544617, "metrics/particle/momentum/class_3/accepted/samples": 11967.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2765316069126129, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5609759092330933, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6181819438934326, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4939714670181274, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14514045417308807, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.930623054504395, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00843365304172039, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.47802823781967163, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04505383223295212, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.365509033203125, "metrics/particle/momentum/class_3/assigned/samples": 14703.0, "metrics/particle/momentum/class_3/efficiency": 0.7878727912902832, "metrics/particle/momentum/class_3/event/energy_bias": 6.030672073364258, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.9756441116333, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.123711585998535, "metrics/particle/momentum/class_3/event/px_bias": 0.15061217546463013, "metrics/particle/momentum/class_3/event/py_bias": -0.5319039821624756, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.325438976287842, "metrics/particle/momentum/class_3/event/pz_bias": 1.3785934448242188, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.607325553894043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011608999222517014, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10654864460229874, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10759702324867249, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.054114699363708496, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15854346752166748, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010309278033673763, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.505308210849762, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1291133016347885, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6682085990905762, "metrics/particle/momentum/class_4/accepted/samples": 291.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.20427270233631134, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.6721261739730835, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6604026556015015, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.608972430229187, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.320718765258789, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5884778499603271, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.13590849936008453, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7400435209274292, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8220338821411133, "metrics/particle/momentum/class_4/event/energy_bias": -1.456412672996521, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7795841693878174, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3447582721710205, "metrics/particle/momentum/class_4/event/px_bias": -0.16746431589126587, "metrics/particle/momentum/class_4/event/py_bias": 0.11278526484966278, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9530658721923828, "metrics/particle/momentum/class_4/event/pz_bias": 0.4191659092903137, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.576617479324341, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006167776416987181, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025051459670066833, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024641461670398712, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012049209326505661, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07022548466920853, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14156495034694672, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.00289318454451859, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.46945640444755554, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03119438886642456, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.12409761548042297, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.12495674937963486, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08919213712215424, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1138226985931396, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.24095766246318817, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.09232534468173981, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2841233015060425, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.14392602443695068, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7335155606269836, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6046806573867798, "metrics/particle/momentum/class_5/event/px_bias": 0.0855991542339325, "metrics/particle/momentum/class_5/event/py_bias": 0.01543380692601204, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.31757721304893494, "metrics/particle/momentum/class_5/event/pz_bias": 0.07893482595682144, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4188045263290405, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4798166751861572, "step_valid_time": 7876.975657701492, "step_total_time": 7878.455474615097, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050185898356607, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07661105366796428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9612733836560552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9619047619047619, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9615889691398556, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0380952380952381, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.957663275352806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9582922824302135, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9579776756401839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9612733836560552, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036895821566507, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08525849876959457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2284, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582967515364355, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9557793345008757, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9570363875493205, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044220665499124345, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552238805970149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9527145359019265, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539675580885576, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582967515364355, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0061987913045523, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07798431591792698, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9342403628117913, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9360090874668686, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06575963718820861, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9327286470143613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9344945096554336, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377845220030349} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_5000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..f982ef6a9998a76f445453cd3a3e356e5be8ab6a --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23683213338385753, "train": {"Classification_binary": 3.8422372341156006, "Classification": 0.029552752152085304, "Regression_pt": 0.21177080273628235, "Regression_eta": 0.011780238710343838, "Regression_sin_phi": 0.007810755167156458, "Regression_cos_phi": 0.008052228018641472, "Regression_energy": 0.2146451622247696, "Total": 11.604799270629883, "Total_unweighted": 4.32584917312488}, "train_group": {}, "valid": {"Classification_binary": 3.919619878133138, "Classification": 0.029651698966821034, "Regression_pt": 0.21889360745747885, "Regression_eta": 0.015327828625837961, "Regression_sin_phi": 0.009158210828900337, "Regression_cos_phi": 0.009548444921771685, "Regression_energy": 0.22440463304519653, "Total": 12.260080973307291, "Total_unweighted": 4.426604301979145, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.029520418494939804, "diagnostic/loss/cld_pf/Classification_binary": 3.8997039794921875, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.009168511256575584, "diagnostic/loss/cld_pf/Regression_energy": 0.2229958474636078, "diagnostic/loss/cld_pf/Regression_eta": 0.014556726440787315, "diagnostic/loss/cld_pf/Regression_pt": 0.21872319281101227, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.008591675199568272, "diagnostic/regression/cld_pf/energy_pred_mean": 0.11927682906389236, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2531493306159973, "diagnostic/regression/cld_pf/energy_residual_mean": 0.09215755760669708, "diagnostic/regression/cld_pf/energy_residual_rms": 0.524132251739502, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.08136534690856934, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2517823874950409, "diagnostic/regression/cld_pf/pt_residual_mean": 0.10853248834609985, "diagnostic/regression/cld_pf/pt_residual_rms": 0.522987425327301, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.8967955708503723, "metrics/particle/class_1/pid_accuracy": 0.9786506295204163, "metrics/particle/class_2/efficiency": 0.2742394804954529, "metrics/particle/class_2/pid_accuracy": 0.4299958348274231, "metrics/particle/class_3/efficiency": 0.6871923208236694, "metrics/particle/class_3/pid_accuracy": 0.9795330762863159, "metrics/particle/class_4/efficiency": 0.8516649603843689, "metrics/particle/class_4/pid_accuracy": 0.15521326661109924, "metrics/particle/class_5/efficiency": 0.9359223246574402, "metrics/particle/class_5/pid_accuracy": 0.30912864208221436, "metrics/particle/count/bias_mean": -0.26274511218070984, "metrics/particle/count/mae": 4.627450942993164, "metrics/particle/count/prediction_mean": 70.3026123046875, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.08734092116355896, "metrics/particle/event/energy_response_mean": 1.008394479751587, "metrics/particle/event/met_abs_error": 9.868402481079102, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07179705798625946, "metrics/particle/event/scalar_pt_response_mean": 1.011111855506897, "metrics/particle/event/vector_pt_closure": 0.07820960879325867, "metrics/particle/matched/delta_eta_abs_mean": 0.01661013253033161, "metrics/particle/matched/delta_phi_abs_mean": 0.013814417645335197, "metrics/particle/matched/delta_r_mean": 0.02414543740451336, "metrics/particle/matched/energy_relative_abs_mean": 0.10210825502872467, "metrics/particle/matched/pid_accuracy": 0.9504010081291199, "metrics/particle/matched/pt_relative_abs_mean": 0.09770115464925766, "metrics/particle/matching/duplicate_fraction": 0.04743266850709915, "metrics/particle/matching/efficiency": 0.757615864276886, "metrics/particle/matching/f1": 0.759028971195221, "metrics/particle/matching/purity": 0.7604473829269409, "metrics/particle/matching/target_coverage_dr0p05": 0.7083499431610107, "metrics/particle/matching/target_coverage_dr0p10": 0.800963282585144, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6469318866729736, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8058444857597351, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.028775466606020927, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.10240893065929413, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09782376140356064, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.033783700317144394, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15058420598506927, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006001909729093313, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5109624266624451, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.012764250859618187, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8810797929763794, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7331.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.41991811990737915, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5109819173812866, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7351148128509521, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6043269634246826, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1847323477268219, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.990846633911133, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.021781567484140396, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1177351474761963, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02004339173436165, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6882407665252686, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9733.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7231922745704651, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.8884036540985107, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.13380241394043, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.125777244567871, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3057442605495453, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.060456037521362305, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.486383438110352, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7150731086730957, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.146244049072266, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.041654955595731735, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12991368770599365, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.11924029886722565, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0456245131790638, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1749677211046219, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026905829086899757, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3522253036499023, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.01882377453148365, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.080601692199707, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1338.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9581469893455505, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.3111331462860107, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8983.1484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8982.9912109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3390989601612091, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 447655.40625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09493161737918854, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.6752171516418457, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09720603376626968, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.589264869689941, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2486.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4867224395275116, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 3.666656494140625, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.25174331665039, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7562295198440552, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11357629299163818, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10287118703126907, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.021716117858887, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.6042191982269287, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.19563102722168, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02102503553032875, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10116399824619293, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09791834652423859, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.025253646075725555, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15124037861824036, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006500189774669707, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1462632268667221, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.013081147335469723, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5751945972442627, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18461.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23566032946109772, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9779115319252014, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7993361949920654, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.66306471824646, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13588881492614746, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.044883728027344, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006097287405282259, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.423310786485672, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.022273410111665726, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8930753469467163, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22305.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8074618577957153, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.5397887229919434, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.775152206420898, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.934126853942871, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.2902860641479492, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1313451528549194, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.303260803222656, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.0532924085855484, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.569637298583984, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02413676492869854, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.10291828215122223, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09894434362649918, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02856326475739479, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15232157707214355, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003391080070286989, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3042870759963989, "metrics/particle/momentum/all/accepted/pz_bias": 0.013278731144964695, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4150956869125366, "metrics/particle/momentum/all/accepted/samples": 27130.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.33963102102279663, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2241971492767334, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 647.8631591796875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 647.726806640625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16429150104522705, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 120125.1171875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01691576838493347, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8532456755638123, "metrics/particle/momentum/all/assigned/pz_bias": 0.02704046294093132, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.573759078979492, "metrics/particle/momentum/all/assigned/samples": 34524.0, "metrics/particle/momentum/all/efficiency": 0.758902370929718, "metrics/particle/momentum/all/event/energy_bias": 5.015271186828613, "metrics/particle/momentum/all/event/p3_error_mean": 24.32682991027832, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.560439109802246, "metrics/particle/momentum/all/event/px_bias": 1.709606647491455, "metrics/particle/momentum/all/event/py_bias": -1.2946723699569702, "metrics/particle/momentum/all/event/pz_abs_mean": 17.714725494384766, "metrics/particle/momentum/all/event/pz_bias": 3.2659997940063477, "metrics/particle/momentum/all/event/pz_residual_rms": 26.923664093017578, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.018067244440317154, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06288542598485947, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.057204470038414, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02555958926677704, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10660211741924286, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016930389683693647, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.21420246362686157, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.013482144102454185, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.968094527721405, "metrics/particle/momentum/class_1/accepted/samples": 15357.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14504259824752808, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4764549434185028, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.60826575756073, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5393565893173218, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08100852370262146, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.606864929199219, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008579432964324951, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5120164155960083, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0006959037273190916, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6441023349761963, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.8958175182342529, "metrics/particle/momentum/class_1/event/energy_bias": 8.26837158203125, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.072877883911133, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.649046897888184, "metrics/particle/momentum/class_1/event/px_bias": 0.1062859520316124, "metrics/particle/momentum/class_1/event/py_bias": -0.06336710602045059, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.832592010498047, "metrics/particle/momentum/class_1/event/pz_bias": 1.5648530721664429, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.173261642456055, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.044721003621816635, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2205159217119217, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21874554455280304, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06911848485469818, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2580021321773529, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.06806930899620056, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.538792371749878, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13290485739707947, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.630871772766113, "metrics/particle/momentum/class_2/accepted/samples": 808.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.266684651374817, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.0390167236328125, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 9336.31640625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 9335.6962890625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4872438311576843, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 456461.96875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.10999581962823868, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.145464897155762, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.017262455075979233, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.792906761169434, "metrics/particle/momentum/class_2/assigned/samples": 2391.0, "metrics/particle/momentum/class_2/efficiency": 0.27958476543426514, "metrics/particle/momentum/class_2/event/energy_bias": -15.125686645507812, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.176090240478516, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.532873153686523, "metrics/particle/momentum/class_2/event/px_bias": 0.49788182973861694, "metrics/particle/momentum/class_2/event/py_bias": -0.530784547328949, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.577548027038574, "metrics/particle/momentum/class_2/event/pz_bias": 0.5767379999160767, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.47856903076172, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.031665876507759094, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1532348394393921, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15155665576457977, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04059125483036041, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19335821270942688, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000761469651479274, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2544194161891937, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.019448041915893555, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0876106023788452, "metrics/particle/momentum/class_3/accepted/samples": 10506.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.42218217253685, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.141097068786621, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.252896785736084, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.114636182785034, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21144278347492218, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.298133850097656, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011677722446620464, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7183020710945129, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.05576242133975029, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0337493419647217, "metrics/particle/momentum/class_3/assigned/samples": 14472.0, "metrics/particle/momentum/class_3/efficiency": 0.6916847825050354, "metrics/particle/momentum/class_3/event/energy_bias": 15.480133056640625, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.317180633544922, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.615272521972656, "metrics/particle/momentum/class_3/event/px_bias": 1.1929806470870972, "metrics/particle/momentum/class_3/event/py_bias": -0.9164063930511475, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.258964538574219, "metrics/particle/momentum/class_3/event/pz_bias": 0.8329411745071411, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.105741500854492, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020859122276306152, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11073428392410278, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10740362852811813, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03221751004457474, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16081638634204865, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010135134682059288, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6297363042831421, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11643044650554657, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.4661900997161865, "metrics/particle/momentum/class_4/accepted/samples": 296.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22596469521522522, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7389780282974243, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8425629138946533, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7647825479507446, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.635843276977539, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8104808330535889, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.16292348504066467, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7819583415985107, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8361582159996033, "metrics/particle/momentum/class_4/event/energy_bias": -2.5772979259490967, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0832109451293945, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1219258308410645, "metrics/particle/momentum/class_4/event/px_bias": 0.07519998401403427, "metrics/particle/momentum/class_4/event/py_bias": 0.2037447988986969, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8122345209121704, "metrics/particle/momentum/class_4/event/pz_bias": 0.15153764188289642, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.761374473571777, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014608511701226234, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.034368596971035004, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.031158683821558952, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013880211859941483, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06635373085737228, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3381870687007904, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.13379918038845062, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.735523521900177, "metrics/particle/momentum/class_5/accepted/samples": 163.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.12604618072509766, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.08530079573392868, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0823468342423439, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.007573665119707584, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05202312022447586, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2777567207813263, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5235788226127625, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.09062445908784866, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.418357014656067, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9421965479850769, "metrics/particle/momentum/class_5/event/energy_bias": -1.030247449874878, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9348270893096924, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4420695304870605, "metrics/particle/momentum/class_5/event/px_bias": -0.16274192929267883, "metrics/particle/momentum/class_5/event/py_bias": 0.012140877544879913, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.044264554977417, "metrics/particle/momentum/class_5/event/pz_bias": 0.139929860830307, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.103587627410889, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.083552360534668, "step_valid_time": 1086.8113043308258, "step_total_time": 1087.894856929779, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0094060855453668, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15391634794315046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.929110600590745, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.930637738330046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9298735424536049, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06936226166995398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9136855923859534, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9151873767258383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9144358679586139, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.929110600590745, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0112351258087198, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.167550508229228, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2270, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2070, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2037, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9086918349429324, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9118942731277533, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9102902374670184, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0881057268722467, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8942054433713784, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8973568281938326, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.895778364116095, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9086918349429324, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.015894229284782, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17875591955678927, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1167, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1145, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8854324734446131, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8847611827141774, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8850967007963595, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1152388172858226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8687405159332322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.868081880212282, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8684110731892302, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8854324734446131} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_50000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..08a38a721c60f098cf0db63b632004c66c5c88c6 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3683213338385753, "train": {"Classification_binary": 2.492763042449951, "Classification": 0.017656970769166946, "Regression_pt": 0.1312498301267624, "Regression_eta": 0.013577324338257313, "Regression_sin_phi": 0.007351048290729523, "Regression_cos_phi": 0.006903244182467461, "Regression_energy": 0.13750657439231873, "Total": 7.969515800476074, "Total_unweighted": 2.8070080345496535}, "train_group": {}, "valid": {"Classification_binary": 2.5264304478963218, "Classification": 0.017827987670898438, "Regression_pt": 0.1313259800275167, "Regression_eta": 0.011394852151473364, "Regression_sin_phi": 0.006601329271992047, "Regression_cos_phi": 0.006969043364127477, "Regression_energy": 0.13704139987627664, "Total": 7.841417948404948, "Total_unweighted": 2.8375910402586064, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01777631603181362, "diagnostic/loss/cld_pf/Classification_binary": 2.5246312618255615, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006716683506965637, "diagnostic/loss/cld_pf/Regression_energy": 0.13777592778205872, "diagnostic/loss/cld_pf/Regression_eta": 0.010432425886392593, "diagnostic/loss/cld_pf/Regression_pt": 0.1327820122241974, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006233794614672661, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07613643258810043, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19363446533679962, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04901714622974396, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41665858030319214, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03981738165020943, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18776996433734894, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06698451936244965, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41073745489120483, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9385372996330261, "metrics/particle/class_1/pid_accuracy": 0.9863592982292175, "metrics/particle/class_2/efficiency": 0.4575595259666443, "metrics/particle/class_2/pid_accuracy": 0.7609561681747437, "metrics/particle/class_3/efficiency": 0.7842167615890503, "metrics/particle/class_3/pid_accuracy": 0.9845873713493347, "metrics/particle/class_4/efficiency": 0.8173562288284302, "metrics/particle/class_4/pid_accuracy": 0.35679012537002563, "metrics/particle/class_5/efficiency": 0.9669902920722961, "metrics/particle/class_5/pid_accuracy": 0.6465863585472107, "metrics/particle/count/bias_mean": -0.29411765933036804, "metrics/particle/count/mae": 3.215686321258545, "metrics/particle/count/prediction_mean": 70.271240234375, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05067402869462967, "metrics/particle/event/energy_response_mean": 0.9879788756370544, "metrics/particle/event/met_abs_error": 4.993486404418945, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040749285370111465, "metrics/particle/event/scalar_pt_response_mean": 0.9968059062957764, "metrics/particle/event/vector_pt_closure": 0.0443912036716938, "metrics/particle/matched/delta_eta_abs_mean": 0.0076048048213124275, "metrics/particle/matched/delta_phi_abs_mean": 0.008742392994463444, "metrics/particle/matched/delta_r_mean": 0.013032173737883568, "metrics/particle/matched/energy_relative_abs_mean": 0.09115463495254517, "metrics/particle/matched/pid_accuracy": 0.9680484533309937, "metrics/particle/matched/pt_relative_abs_mean": 0.08463682979345322, "metrics/particle/matching/duplicate_fraction": 0.030916616320610046, "metrics/particle/matching/efficiency": 0.8334182500839233, "metrics/particle/matching/f1": 0.8351587057113647, "metrics/particle/matching/purity": 0.8369064927101135, "metrics/particle/matching/target_coverage_dr0p05": 0.822674036026001, "metrics/particle/matching/target_coverage_dr0p10": 0.8685870170593262, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7284582853317261, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8686055541038513, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01488879881799221, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08687670528888702, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08390811085700989, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020266752690076828, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13644973933696747, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00472097797319293, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38231033086776733, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.025936350226402283, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.553439736366272, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8261.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25017428398132324, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9018982648849487, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.922412097454071, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8158124685287476, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11728958040475845, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.648524284362793, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013673655688762665, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7192000150680542, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.015030032023787498, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.493016004562378, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9873.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8149353861808777, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.38517245650291443, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.165054798126221, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.793966054916382, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1847284883260727, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.10536317527294159, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.267951965332031, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4714576005935669, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.991013526916504, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023093925788998604, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09477749466896057, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09620967507362366, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02294265851378441, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14798368513584137, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022924412041902542, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9465599656105042, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18399052321910858, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0597524642944336, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1614.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7792017459869385, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.743762493133545, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 29603.798828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 29603.63671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2655212879180908, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1497989.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08004685491323471, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8441145420074463, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08469831943511963, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.240446090698242, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2561.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5871226191520691, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0913498401641846, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.835669040679932, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.021133303642273, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03375514596700668, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06154979392886162, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.682823657989502, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.011695444583892822, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.338677406311035, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01131353061646223, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0941530093550682, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08610040694475174, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01599130406975746, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14089435338974, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005019576638005674, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11361955851316452, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0013369123917073011, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5131285190582275, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19922.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1383771002292633, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6414800882339478, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5125572681427002, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4019728899002075, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08908160775899887, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.250438213348389, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002579497406259179, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24618875980377197, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.009833055548369884, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4886775016784668, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22485.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8713642358779907, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.757054090499878, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.669958114624023, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.674745559692383, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.056427545845508575, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.029559075832366943, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9262502193450928, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2215476930141449, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.729547500610352, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012942850589752197, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09216953068971634, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0860401913523674, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017553171142935753, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14007282257080078, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028861965984106064, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2332295924425125, "metrics/particle/momentum/all/accepted/pz_bias": -0.016262944787740707, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.162835955619812, "metrics/particle/momentum/all/accepted/samples": 29797.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21698544919490814, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7959534525871277, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2171.767578125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2171.654296875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10999742150306702, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 405679.21875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011397805996239185, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5704628825187683, "metrics/particle/momentum/all/assigned/pz_bias": -0.004129776265472174, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.857362747192383, "metrics/particle/momentum/all/assigned/samples": 34919.0, "metrics/particle/momentum/all/efficiency": 0.8335058093070984, "metrics/particle/momentum/all/event/energy_bias": -2.4632315635681152, "metrics/particle/momentum/all/event/p3_error_mean": 13.80346393585205, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.208185195922852, "metrics/particle/momentum/all/event/px_bias": 0.27491119503974915, "metrics/particle/momentum/all/event/py_bias": 0.19647204875946045, "metrics/particle/momentum/all/event/pz_abs_mean": 9.996126174926758, "metrics/particle/momentum/all/event/pz_bias": -0.23821449279785156, "metrics/particle/momentum/all/event/pz_residual_rms": 17.349822998046875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008130894042551517, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05205569043755531, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04079670086503029, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00820625014603138, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0904322937130928, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015574382850900292, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1449992060661316, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0056854113936424255, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8740931749343872, "metrics/particle/momentum/class_1/accepted/samples": 16052.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0839904397726059, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24973243474960327, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3017643094062805, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24585023522377014, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04475957155227661, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.378483772277832, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005660597700625658, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31190791726112366, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.005020488053560257, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2502031326293945, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9363588690757751, "metrics/particle/momentum/class_1/event/energy_bias": 1.8017983436584473, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.207175254821777, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.638712406158447, "metrics/particle/momentum/class_1/event/px_bias": 0.4293057322502136, "metrics/particle/momentum/class_1/event/py_bias": 0.14058290421962738, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.403155326843262, "metrics/particle/momentum/class_1/event/pz_bias": -0.41373783349990845, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.939495086669922, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032511815428733826, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19661225378513336, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19995462894439697, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04566529765725136, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23814357817173004, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.036226414144039154, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.718335747718811, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.27666711807250977, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.032841682434082, "metrics/particle/momentum/class_2/accepted/samples": 1325.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8437750339508057, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7443099021911621, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 29012.75390625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 29012.294921875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35667815804481506, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1483009.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07386146485805511, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9433324337005615, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.10248500853776932, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.754136085510254, "metrics/particle/momentum/class_2/assigned/samples": 2613.0, "metrics/particle/momentum/class_2/efficiency": 0.4584774971008301, "metrics/particle/momentum/class_2/event/energy_bias": -8.549457550048828, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.639737129211426, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.763146877288818, "metrics/particle/momentum/class_2/event/px_bias": -0.10121113061904907, "metrics/particle/momentum/class_2/event/py_bias": 0.21927094459533691, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.126948356628418, "metrics/particle/momentum/class_2/event/pz_bias": -1.6543693542480469, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.817584991455078, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017357416450977325, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1349135935306549, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13437619805335999, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03622779622673988, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1757953017950058, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009190408745780587, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1823173612356186, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006840371992439032, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6929604411125183, "metrics/particle/momentum/class_3/accepted/samples": 11969.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2613862454891205, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4048243761062622, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3431662321090698, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.221741795539856, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14197909832000732, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.082860946655273, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007034077774733305, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4507930278778076, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.009937150403857231, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.297227144241333, "metrics/particle/momentum/class_3/assigned/samples": 14643.0, "metrics/particle/momentum/class_3/efficiency": 0.788004457950592, "metrics/particle/momentum/class_3/event/energy_bias": 5.613992691040039, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.969481468200684, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.940185546875, "metrics/particle/momentum/class_3/event/px_bias": 0.12143008410930634, "metrics/particle/momentum/class_3/event/py_bias": -0.30547335743904114, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.445720672607422, "metrics/particle/momentum/class_3/event/pz_bias": 1.2811028957366943, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.35498046875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011361632496118546, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11071531474590302, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1113252341747284, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.056985653936862946, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16380034387111664, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007042253389954567, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.488282173871994, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1716168075799942, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5587095022201538, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28535348176956177, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.7589476108551025, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1764824390411377, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.121809244155884, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.777059555053711, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6896505355834961, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.18103277683258057, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.201056957244873, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.3195419311523438, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7777259349822998, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3035497665405273, "metrics/particle/momentum/class_4/event/px_bias": -0.27926599979400635, "metrics/particle/momentum/class_4/event/py_bias": 0.08077901601791382, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9612915515899658, "metrics/particle/momentum/class_4/event/pz_bias": 0.4556676745414734, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.202303409576416, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006498376838862896, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024205168709158897, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023747656494379044, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012079501524567604, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06484203785657883, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14605532586574554, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03827812150120735, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.48806488513946533, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.025300370529294014, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.12767651677131653, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.12845638394355774, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0933404192328453, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1603710651397705, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2260911464691162, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.0001240045385202393, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8986412882804871, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.010022595524787903, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7779959440231323, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6516602635383606, "metrics/particle/momentum/class_5/event/px_bias": 0.1046525239944458, "metrics/particle/momentum/class_5/event/py_bias": 0.06131253391504288, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.32313072681427, "metrics/particle/momentum/class_5/event/pz_bias": 0.0931222140789032, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4666308164596558, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3546826839447021, "step_valid_time": 8018.547900676727, "step_total_time": 8019.902583599091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017870612297255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07500968425176968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2926, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9666996373227827, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9644736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033300362677217275, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602888086642599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9647213979558193, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008944059491456, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08443664588542399, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2287, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2184, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2177, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9587357330992098, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954962833406209, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9568455640744797, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04503716659379099, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519020550940096, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9537787513691128, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9587357330992098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0022301214410982, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08353369172455327, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9364118092354277, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9374763167866617, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0635881907645723, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9364118092354277, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9374763167866617, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385432473444613} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_6000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..8e5599e8435773625f7b3587b8a6f784624df0e9 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28419856006062905, "train": {"Classification_binary": 3.8348758220672607, "Classification": 0.029746172949671745, "Regression_pt": 0.20352518558502197, "Regression_eta": 0.011822138912975788, "Regression_sin_phi": 0.009271876886487007, "Regression_cos_phi": 0.009267181158065796, "Regression_energy": 0.20744240283966064, "Total": 11.646998405456543, "Total_unweighted": 4.305950780399144}, "train_group": {}, "valid": {"Classification_binary": 3.788911819458008, "Classification": 0.0278695747256279, "Regression_pt": 0.20756012201309204, "Regression_eta": 0.014747358858585358, "Regression_sin_phi": 0.008919411028424898, "Regression_cos_phi": 0.009499915565053621, "Regression_energy": 0.2142192522684733, "Total": 11.764629364013672, "Total_unweighted": 4.271727453917265, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.027897968888282776, "diagnostic/loss/cld_pf/Classification_binary": 3.773671865463257, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.009023410268127918, "diagnostic/loss/cld_pf/Regression_energy": 0.21471014618873596, "diagnostic/loss/cld_pf/Regression_eta": 0.013963493518531322, "diagnostic/loss/cld_pf/Regression_pt": 0.20961172878742218, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00833092536777258, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08936561644077301, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.23971575498580933, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06224633380770683, "diagnostic/regression/cld_pf/energy_residual_rms": 0.50233393907547, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.060803137719631195, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.23894470930099487, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08797027915716171, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5010802745819092, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9056115746498108, "metrics/particle/class_1/pid_accuracy": 0.979874312877655, "metrics/particle/class_2/efficiency": 0.27344194054603577, "metrics/particle/class_2/pid_accuracy": 0.4570833444595337, "metrics/particle/class_3/efficiency": 0.683742105960846, "metrics/particle/class_3/pid_accuracy": 0.9791727066040039, "metrics/particle/class_4/efficiency": 0.8577194809913635, "metrics/particle/class_4/pid_accuracy": 0.1823529452085495, "metrics/particle/class_5/efficiency": 0.955339789390564, "metrics/particle/class_5/pid_accuracy": 0.40040650963783264, "metrics/particle/count/bias_mean": -1.0895425081253052, "metrics/particle/count/mae": 4.489542484283447, "metrics/particle/count/prediction_mean": 69.47581481933594, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.07976463437080383, "metrics/particle/event/energy_response_mean": 0.9718689322471619, "metrics/particle/event/met_abs_error": 8.577729225158691, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06582053005695343, "metrics/particle/event/scalar_pt_response_mean": 0.9842584133148193, "metrics/particle/event/vector_pt_closure": 0.07174131274223328, "metrics/particle/matched/delta_eta_abs_mean": 0.014541410841047764, "metrics/particle/matched/delta_phi_abs_mean": 0.015387244522571564, "metrics/particle/matched/delta_r_mean": 0.023749645799398422, "metrics/particle/matched/energy_relative_abs_mean": 0.09706173092126846, "metrics/particle/matched/pid_accuracy": 0.9525989890098572, "metrics/particle/matched/pt_relative_abs_mean": 0.09319010376930237, "metrics/particle/matching/duplicate_fraction": 0.043321605771780014, "metrics/particle/matching/efficiency": 0.760505735874176, "metrics/particle/matching/f1": 0.7664225697517395, "metrics/particle/matching/purity": 0.7724322080612183, "metrics/particle/matching/target_coverage_dr0p05": 0.6938174366950989, "metrics/particle/matching/target_coverage_dr0p10": 0.7998610734939575, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6573426723480225, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8111425042152405, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02622951567173004, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09267815947532654, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09215329587459564, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024654876440763474, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1449505239725113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00679994560778141, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.47190552949905396, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.12002956867218018, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9079700708389282, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7353.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4004332423210144, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.39342200756073, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4529719352722168, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.317943811416626, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17571620643138885, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.1641206741333, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022132588550448418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0662466287612915, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1776828169822693, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7639851570129395, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9669.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7253625392913818, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.2181121110916138, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.224002838134766, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.062487602233887, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.22038590908050537, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.4884660243988037, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.613036155700684, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.209643840789795, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.395305633544922, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03389611840248108, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.11082182824611664, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.11066542565822601, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03791012987494469, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16644369065761566, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0189825352281332, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0146647691726685, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2291886806488037, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0972959995269775, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1317.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.973770022392273, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.5415284633636475, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 16571.693359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 16571.5234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3479868471622467, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 817338.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0895645022392273, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.646775245666504, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.9108383059501648, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.478862762451172, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2434.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4790832996368408, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.3508310317993164, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.050325393676758, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.60880708694458, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.18150116503238678, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.20552082359790802, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.847028732299805, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -6.998173236846924, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.05210304260254, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022023918107151985, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09898428618907928, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09440546482801437, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01600644178688526, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1495589315891266, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008093233918771148, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14068876206874847, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.021208658814430237, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6150140762329102, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18534.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22670117020606995, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8480132222175598, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6826025247573853, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5420824289321899, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12604208290576935, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.751547336578369, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005813167430460453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.38825106620788574, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04296879470348358, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9893815517425537, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22191.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8106547594070435, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.7708659172058105, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.226547241210938, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.45281982421875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.17266608774662018, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7781183123588562, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.090056419372559, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6975699067115784, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.014954566955566, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023735411465168, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09785287827253342, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0945839062333107, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019404439255595207, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14919564127922058, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033083369489759207, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2725246250629425, "metrics/particle/momentum/all/accepted/pz_bias": -0.05798780173063278, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.306169033050537, "metrics/particle/momentum/all/accepted/samples": 27204.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3287068009376526, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1219844818115234, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1177.0191650390625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1176.8782958984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15579985082149506, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 217747.609375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016358546912670135, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8106802701950073, "metrics/particle/momentum/all/assigned/pz_bias": -0.14254726469516754, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.988394260406494, "metrics/particle/momentum/all/assigned/samples": 34294.0, "metrics/particle/momentum/all/efficiency": 0.7609723210334778, "metrics/particle/momentum/all/event/energy_bias": -6.339808940887451, "metrics/particle/momentum/all/event/p3_error_mean": 23.708518981933594, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.176700592041016, "metrics/particle/momentum/all/event/px_bias": 0.22922097146511078, "metrics/particle/momentum/all/event/py_bias": 1.4721052646636963, "metrics/particle/momentum/all/event/pz_abs_mean": 17.51093864440918, "metrics/particle/momentum/all/event/pz_bias": -10.905385971069336, "metrics/particle/momentum/all/event/pz_residual_rms": 27.46023941040039, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014834307134151459, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05753595381975174, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.05128086730837822, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.014080801047384739, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10100898891687393, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018093700055032969, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17424459755420685, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04142766073346138, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9164461493492126, "metrics/particle/momentum/class_1/accepted/samples": 15475.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13798362016677856, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3372586965560913, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.41914308071136475, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.34192702174186707, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07259146869182587, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.398246765136719, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0088113434612751, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4452257752418518, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07292655110359192, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.586864948272705, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9027007818222046, "metrics/particle/momentum/class_1/event/energy_bias": 4.238857746124268, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.7519474029541, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.895435333251953, "metrics/particle/momentum/class_1/event/px_bias": 0.44119638204574585, "metrics/particle/momentum/class_1/event/py_bias": 0.4138457775115967, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.352046966552734, "metrics/particle/momentum/class_1/event/pz_bias": -3.2136971950531006, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.269575119018555, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04399019479751587, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2300204038619995, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2281101644039154, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08175534009933472, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2648429870605469, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.05415617302060127, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.3441696166992188, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.424836665391922, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.054473400115967, "metrics/particle/momentum/class_2/accepted/samples": 794.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.2154916524887085, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9864447712898254, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17487.716796875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17487.08984375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4852558672428131, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 839716.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.10060711205005646, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.065113067626953, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.5207306742668152, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.902851104736328, "metrics/particle/momentum/class_2/assigned/samples": 2306.0, "metrics/particle/momentum/class_2/efficiency": 0.2747404873371124, "metrics/particle/momentum/class_2/event/energy_bias": -17.253467559814453, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.10485076904297, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.285043716430664, "metrics/particle/momentum/class_2/event/px_bias": 0.2579832077026367, "metrics/particle/momentum/class_2/event/py_bias": 0.5623282790184021, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.55898666381836, "metrics/particle/momentum/class_2/event/pz_bias": -3.8122262954711914, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.282981872558594, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03573895990848541, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14817063510417938, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14911724627017975, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03494459390640259, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19139444828033447, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015286137349903584, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24903962016105652, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05368909239768982, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0305299758911133, "metrics/particle/momentum/class_3/accepted/samples": 10467.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4214884042739868, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0596699714660645, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0908679962158203, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9492465257644653, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20474694669246674, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.947050094604492, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011937173083424568, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7266284227371216, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.1662800908088684, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.123724460601807, "metrics/particle/momentum/class_3/assigned/samples": 14325.0, "metrics/particle/momentum/class_3/efficiency": 0.6891171336174011, "metrics/particle/momentum/class_3/event/energy_bias": 9.247930526733398, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.876527786254883, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.3128743171691895, "metrics/particle/momentum/class_3/event/px_bias": -0.36221471428871155, "metrics/particle/momentum/class_3/event/py_bias": 0.10232802480459213, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.200140953063965, "metrics/particle/momentum/class_3/event/pz_bias": -4.255291938781738, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.22164535522461, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017220307141542435, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10546699166297913, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10459964722394943, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.02662099152803421, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15687088668346405, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009966777637600899, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6734569072723389, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.07113475352525711, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.5078537464141846, "metrics/particle/momentum/class_4/accepted/samples": 301.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.17761529982089996, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.5707857608795166, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.486504554748535, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.4071438312530518, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11898016929626465, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.46632957458496, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016997167840600014, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9189871549606323, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0898425281047821, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.4480221271514893, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8502824902534485, "metrics/particle/momentum/class_4/event/energy_bias": -2.0127477645874023, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.3836591243743896, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.3951973915100098, "metrics/particle/momentum/class_4/event/px_bias": -0.12380477786064148, "metrics/particle/momentum/class_4/event/py_bias": 0.39049696922302246, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9252855777740479, "metrics/particle/momentum/class_4/event/pz_bias": 0.13805542886257172, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.039721965789795, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01165267638862133, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.037945326417684555, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03637748956680298, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006668927613645792, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08223289251327515, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.279317170381546, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.094080850481987, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8521178364753723, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02659967541694641, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.062064770609140396, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06120980530977249, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01575627364218235, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1777203381061554, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.37069809436798096, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.1404179185628891, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.262560248374939, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.5603841543197632, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7417421340942383, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2851078510284424, "metrics/particle/momentum/class_5/event/px_bias": 0.01606087014079094, "metrics/particle/momentum/class_5/event/py_bias": 0.0031063035130500793, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.946762204170227, "metrics/particle/momentum/class_5/event/pz_bias": 0.23777326941490173, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.080249309539795, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.1908397674560547, "step_valid_time": 1286.767810344696, "step_total_time": 1287.9586503505707, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9900838327415634, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.138810709756792, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2819, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2800, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9251723006235641, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9294427959116386, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9273026315789473, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07055720408836136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9189366590088611, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9231783712495879, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9210526315789473, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9251723006235641, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9883881565267773, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1552331869177277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2054, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9117647058823529, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9161887957653286, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.913971397139714, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08381120423467137, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9016681299385426, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.906043228936921, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9038503850385039, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9117647058823529, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9861400716073224, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14931155948553865, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1301, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1149, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1138, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8717754172989378, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8831667947732513, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8774341351660939, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11683320522674866, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8634294385432474, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8747117601844735, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8690339824360442, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8717754172989378} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_7000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..abae5b7363de7af0e274ee84b9c7131ae3173c61 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.33156498673740054, "train": {"Classification_binary": 3.656247615814209, "Classification": 0.025374487042427063, "Regression_pt": 0.240628182888031, "Regression_eta": 0.013752613216638565, "Regression_sin_phi": 0.010426739230751991, "Regression_cos_phi": 0.011348884552717209, "Regression_energy": 0.24678169190883636, "Total": 12.505236625671387, "Total_unweighted": 4.204560214653611}, "train_group": {}, "valid": {"Classification_binary": 3.655059814453125, "Classification": 0.026591360569000244, "Regression_pt": 0.20544201135635376, "Regression_eta": 0.014683540910482407, "Regression_sin_phi": 0.00878498206535975, "Regression_cos_phi": 0.009255640208721161, "Regression_energy": 0.21069061756134033, "Total": 11.487495422363281, "Total_unweighted": 4.130507967124383, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.026679139584302902, "diagnostic/loss/cld_pf/Classification_binary": 3.6421847343444824, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.008893096819519997, "diagnostic/loss/cld_pf/Regression_energy": 0.2114459127187729, "diagnostic/loss/cld_pf/Regression_eta": 0.013828292489051819, "diagnostic/loss/cld_pf/Regression_pt": 0.20788562297821045, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.008159196004271507, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09661143273115158, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.24410410225391388, "diagnostic/regression/cld_pf/energy_residual_mean": 0.0694921612739563, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5083487629890442, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05791328474879265, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2402690201997757, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08508042246103287, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5046167373657227, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9046533107757568, "metrics/particle/class_1/pid_accuracy": 0.9802131056785583, "metrics/particle/class_2/efficiency": 0.31536972522735596, "metrics/particle/class_2/pid_accuracy": 0.5827311873435974, "metrics/particle/class_3/efficiency": 0.6957190632820129, "metrics/particle/class_3/pid_accuracy": 0.9768146872520447, "metrics/particle/class_4/efficiency": 0.8617557883262634, "metrics/particle/class_4/pid_accuracy": 0.16042155027389526, "metrics/particle/class_5/efficiency": 0.9436893463134766, "metrics/particle/class_5/pid_accuracy": 0.395061731338501, "metrics/particle/count/bias_mean": -0.239215686917305, "metrics/particle/count/mae": 4.312418460845947, "metrics/particle/count/prediction_mean": 70.32614135742188, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.07332702726125717, "metrics/particle/event/energy_response_mean": 0.9922608137130737, "metrics/particle/event/met_abs_error": 8.489446640014648, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0647156834602356, "metrics/particle/event/scalar_pt_response_mean": 0.9997612833976746, "metrics/particle/event/vector_pt_closure": 0.07122138887643814, "metrics/particle/matched/delta_eta_abs_mean": 0.012962005101144314, "metrics/particle/matched/delta_phi_abs_mean": 0.015049183741211891, "metrics/particle/matched/delta_r_mean": 0.02216348797082901, "metrics/particle/matched/energy_relative_abs_mean": 0.100643590092659, "metrics/particle/matched/pid_accuracy": 0.9537899494171143, "metrics/particle/matched/pt_relative_abs_mean": 0.09428703039884567, "metrics/particle/matching/duplicate_fraction": 0.04568815603852272, "metrics/particle/matching/efficiency": 0.7684805393218994, "metrics/particle/matching/f1": 0.7697852849960327, "metrics/particle/matching/purity": 0.7710945010185242, "metrics/particle/matching/target_coverage_dr0p05": 0.7129440307617188, "metrics/particle/matching/target_coverage_dr0p10": 0.8109480142593384, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6628074049949646, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8178669214248657, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02484431304037571, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09497752040624619, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0915861576795578, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0015795918880030513, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.143349289894104, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006233421619981527, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4550786316394806, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04762796685099602, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8960282802581787, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7540.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.39840105175971985, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.620205044746399, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.646393895149231, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.4966555833816528, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16976696252822876, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.509536743164062, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.023405559360980988, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.08100163936615, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.11672256141901016, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.8957910537719727, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9784.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7438098192214966, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.6114967465400696, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.217412948608398, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.896557331085205, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.020584719255566597, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.17184217274188995, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.710504531860352, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.3556783199310303, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.731914520263672, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03391190990805626, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10288051515817642, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10354151576757431, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0021946062333881855, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15523043274879456, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02045288495719433, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0208122730255127, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20813025534152985, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4996423721313477, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1369.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9596114754676819, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6172161102294922, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12902.138671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12901.9326171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3267483115196228, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 648919.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09205847233533859, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.5353610515594482, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5377281308174133, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.210782051086426, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2531.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.49799928069114685, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.5718640089035034, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.34239387512207, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4287357330322266, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.12463030964136124, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10206419229507446, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.138091087341309, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.7896642684936523, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.30267906188965, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020521031692624092, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10428562760353088, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09649202227592468, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014741306193172932, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15082204341888428, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005369127611629665, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14035041630268097, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0038457552436739206, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5727084279060364, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18625.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21672025322914124, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9386979937553406, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7453123331069946, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6049126386642456, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13011185824871063, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.600048542022705, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005100671201944351, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.37856000661849976, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0036228387616574764, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9657942056655884, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22350.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8146349787712097, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.26283156871795654, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.605636596679688, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.821293830871582, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.0263117551803589, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8405783176422119, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.9642815589904785, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.08871272206306458, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.669684410095215, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022370733320713043, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.10166679322719574, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09549909085035324, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010294999927282333, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14903821051120758, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003087092423811555, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.27031347155570984, "metrics/particle/momentum/all/accepted/pz_bias": -0.02078949473798275, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.347316861152649, "metrics/particle/momentum/all/accepted/samples": 27534.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3222394585609436, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.180590271949768, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 942.970703125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 942.8228759765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15566132962703705, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 175344.109375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016616184264421463, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8073081374168396, "metrics/particle/momentum/all/assigned/pz_bias": -0.07454128563404083, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.7930908203125, "metrics/particle/momentum/all/assigned/samples": 34665.0, "metrics/particle/momentum/all/efficiency": 0.7702033519744873, "metrics/particle/momentum/all/event/energy_bias": 0.22319889068603516, "metrics/particle/momentum/all/event/p3_error_mean": 22.462980270385742, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.647544860839844, "metrics/particle/momentum/all/event/px_bias": -0.9222661256790161, "metrics/particle/momentum/all/event/py_bias": -0.910356342792511, "metrics/particle/momentum/all/event/pz_abs_mean": 16.710750579833984, "metrics/particle/momentum/all/event/pz_bias": -6.0566301345825195, "metrics/particle/momentum/all/event/pz_residual_rms": 24.89568519592285, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013005377724766731, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06302053481340408, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0527305006980896, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0043762666173279285, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10167625546455383, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019373587565496564, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1838890165090561, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00397080834954977, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.0685888528823853, "metrics/particle/momentum/class_1/accepted/samples": 15485.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1341618299484253, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.36519670486450195, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4378131926059723, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.35376977920532227, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07368727773427963, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.682367324829102, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009451575577259064, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4631725549697876, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.042397696524858475, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.621215343475342, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9032841324806213, "metrics/particle/momentum/class_1/event/energy_bias": 5.304410457611084, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.934288024902344, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.834442138671875, "metrics/particle/momentum/class_1/event/px_bias": 0.006067737936973572, "metrics/particle/momentum/class_1/event/py_bias": 0.051406536251306534, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.5719575881958, "metrics/particle/momentum/class_1/event/pz_bias": -1.2687995433807373, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.73151206970215, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04543304443359375, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21704906225204468, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22004151344299316, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06217081472277641, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25816431641578674, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.046708583831787, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.44773921370506287, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.613253116607666, "metrics/particle/momentum/class_2/accepted/samples": 966.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1620354652404785, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9989180564880371, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 13013.349609375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 13012.7685546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4531686007976532, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 651758.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09884416311979294, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.029864311218262, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.50907963514328, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.606147766113281, "metrics/particle/momentum/class_2/assigned/samples": 2509.0, "metrics/particle/momentum/class_2/efficiency": 0.33425605297088623, "metrics/particle/momentum/class_2/event/energy_bias": -9.835342407226562, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.181827545166016, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.220468521118164, "metrics/particle/momentum/class_2/event/px_bias": -0.9768338203430176, "metrics/particle/momentum/class_2/event/py_bias": 0.3139296770095825, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.786844253540039, "metrics/particle/momentum/class_2/event/pz_bias": -3.194786310195923, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.142133712768555, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03424626588821411, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14843809604644775, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1472121775150299, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02552901953458786, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1887313276529312, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007536504999734461, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22263750433921814, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008056371472775936, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9456673860549927, "metrics/particle/momentum/class_3/accepted/samples": 10615.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4062214195728302, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.1912198066711426, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.052020311355591, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9010295867919922, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20325765013694763, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.58786392211914, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01097384188324213, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6634397506713867, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03709452226758003, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.1830039024353027, "metrics/particle/momentum/class_3/assigned/samples": 14489.0, "metrics/particle/momentum/class_3/efficiency": 0.6988610029220581, "metrics/particle/momentum/class_3/event/energy_bias": 8.311324119567871, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.780771255493164, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.085196495056152, "metrics/particle/momentum/class_3/event/px_bias": 0.019557982683181763, "metrics/particle/momentum/class_3/event/py_bias": -1.298133134841919, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.064249992370605, "metrics/particle/momentum/class_3/event/pz_bias": -1.86671781539917, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.927242279052734, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017666403204202652, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10865437239408493, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10864008963108063, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.01859542354941368, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16536478698253632, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01655629090964794, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7238268256187439, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.03165702149271965, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6793227195739746, "metrics/particle/momentum/class_4/accepted/samples": 302.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.17750184237957, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0986474752426147, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.9253997206687927, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.84198397397995, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12711864709854126, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.365840435028076, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7335308790206909, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05784837156534195, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.5490055084228516, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8531073331832886, "metrics/particle/momentum/class_4/event/energy_bias": -2.7791049480438232, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.148094654083252, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2148828506469727, "metrics/particle/momentum/class_4/event/px_bias": 0.0960356742143631, "metrics/particle/momentum/class_4/event/py_bias": 0.1411406546831131, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8098430633544922, "metrics/particle/momentum/class_4/event/pz_bias": 0.2354159951210022, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.431362628936768, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01096248161047697, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.031730834394693375, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.029598930850625038, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.005140409804880619, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06805132329463959, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2185317575931549, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.014794977381825447, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.694430410861969, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03914463892579079, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1265840381383896, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.12960003316402435, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0831587165594101, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.0151997804641724, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.011560693383216858, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.36634698510169983, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.12748971581459045, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.4379879236221313, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.7780894041061401, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5559971332550049, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1482548713684082, "metrics/particle/momentum/class_5/event/px_bias": -0.06709369271993637, "metrics/particle/momentum/class_5/event/py_bias": -0.11870002746582031, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8484264612197876, "metrics/particle/momentum/class_5/event/pz_bias": 0.03825780376791954, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.690827369689941, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.2412457466125488, "step_valid_time": 1482.7861895561218, "step_total_time": 1484.027435541153, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933154779280253, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13683024530962018, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2810, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9300951755825402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9374793251736685, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9337726523887974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06252067482633146, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9222185756481786, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9295401918623883, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9258649093904449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9300951755825402, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9881144703089132, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15022616115211518, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2269, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2092, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9183494293239683, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9219920669898634, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9201671431713218, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07800793301013663, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9095697980684812, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9131776112825033, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9113701341543875, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9183494293239683, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9905377164212508, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15103279210533593, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1308, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1167, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8968133535660091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9036697247706422, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9002284843869002, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0963302752293578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8854324734446131, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8922018348623854, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8888042650418888, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8968133535660091} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_8000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..23f04e23464c8dcc42f7f97111f76e07e9d41c30 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.378931413414172, "train": {"Classification_binary": 3.487917184829712, "Classification": 0.024923479184508324, "Regression_pt": 0.17490622401237488, "Regression_eta": 0.012098710983991623, "Regression_sin_phi": 0.00766314473003149, "Regression_cos_phi": 0.010129982605576515, "Regression_energy": 0.17701710760593414, "Total": 10.3618803024292, "Total_unweighted": 3.894655833952129}, "train_group": {}, "valid": {"Classification_binary": 3.5129451751708984, "Classification": 0.024721818665663402, "Regression_pt": 0.19242644309997559, "Regression_eta": 0.014302154382069906, "Regression_sin_phi": 0.00837919240196546, "Regression_cos_phi": 0.00887289953728517, "Regression_energy": 0.1986126701037089, "Total": 10.908055623372396, "Total_unweighted": 3.960260353361567, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.025052886456251144, "diagnostic/loss/cld_pf/Classification_binary": 3.5065765380859375, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.008539616130292416, "diagnostic/loss/cld_pf/Regression_energy": 0.19922228157520294, "diagnostic/loss/cld_pf/Regression_eta": 0.013450094498693943, "diagnostic/loss/cld_pf/Regression_pt": 0.19439074397087097, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007734031416475773, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08903592824935913, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2296242117881775, "diagnostic/regression/cld_pf/energy_residual_mean": 0.061916641891002655, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4881548583507538, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05420627072453499, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2259279191493988, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08137340843677521, "diagnostic/regression/cld_pf/pt_residual_rms": 0.48403218388557434, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9140442609786987, "metrics/particle/class_1/pid_accuracy": 0.9795987010002136, "metrics/particle/class_2/efficiency": 0.3384983539581299, "metrics/particle/class_2/pid_accuracy": 0.595085859298706, "metrics/particle/class_3/efficiency": 0.738023042678833, "metrics/particle/class_3/pid_accuracy": 0.9802578687667847, "metrics/particle/class_4/efficiency": 0.8799192905426025, "metrics/particle/class_4/pid_accuracy": 0.18233944475650787, "metrics/particle/class_5/efficiency": 0.9592233300209045, "metrics/particle/class_5/pid_accuracy": 0.45951417088508606, "metrics/particle/count/bias_mean": 1.2104575634002686, "metrics/particle/count/mae": 4.271895408630371, "metrics/particle/count/prediction_mean": 71.77581787109375, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06978139281272888, "metrics/particle/event/energy_response_mean": 0.9880375266075134, "metrics/particle/event/met_abs_error": 7.502100467681885, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05918603017926216, "metrics/particle/event/scalar_pt_response_mean": 1.0019526481628418, "metrics/particle/event/vector_pt_closure": 0.06561640650033951, "metrics/particle/matched/delta_eta_abs_mean": 0.01707504130899906, "metrics/particle/matched/delta_phi_abs_mean": 0.013096400536596775, "metrics/particle/matched/delta_r_mean": 0.02395297773182392, "metrics/particle/matched/energy_relative_abs_mean": 0.09841607511043549, "metrics/particle/matched/pid_accuracy": 0.9553923010826111, "metrics/particle/matched/pt_relative_abs_mean": 0.09294123947620392, "metrics/particle/matching/duplicate_fraction": 0.04351785406470299, "metrics/particle/matching/efficiency": 0.7929699420928955, "metrics/particle/matching/f1": 0.7862265706062317, "metrics/particle/matching/purity": 0.7795969843864441, "metrics/particle/matching/target_coverage_dr0p05": 0.7234566807746887, "metrics/particle/matching/target_coverage_dr0p10": 0.8325197696685791, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6794331669807434, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.836836040019989, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02563127689063549, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09484240412712097, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09210378676652908, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020258748903870583, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1452498883008957, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0063225808553397655, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.45523783564567566, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0014644642360508442, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.790108561515808, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7750.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3675166368484497, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1853184700012207, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3606981039047241, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2231943607330322, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1605374813079834, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.840784072875977, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020711254328489304, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9911770224571228, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.017009317874908447, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5441765785217285, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9898.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7645260095596313, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9152454137802124, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.988028526306152, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.337141990661621, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.19162815809249878, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.03656695783138275, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.680219650268555, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3154432773590088, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.042085647583008, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03587622940540314, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10735975205898285, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10363436490297318, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02492545172572136, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15364111959934235, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02770083025097847, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1262903213500977, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.10064808279275894, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7600362300872803, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1444.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9636420011520386, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8107497692108154, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 54905.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 54905.76171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3125739097595215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2765326.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08474576473236084, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.134169101715088, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3479786813259125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.526342391967773, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2537.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5252819061279297, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.072324514389038, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.743924140930176, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2589844465255737, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09586045891046524, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.011986145749688148, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.57346248626709, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.257478952407837, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.749910354614258, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02257859893143177, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10081212222576141, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09452056139707565, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01828472502529621, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15038350224494934, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008311688434332609, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14185552299022675, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01887962408363819, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6096270680427551, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19250.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20173922181129456, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8414947390556335, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7122989296913147, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5768132209777832, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11885300278663635, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.680192947387695, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0041090440936386585, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33974266052246094, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02370077185332775, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6731033325195312, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22633.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.841971755027771, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.720062732696533, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.986167907714844, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.233575820922852, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3410242199897766, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.8761811852455139, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.743161678314209, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.5358530282974243, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.195724487304688, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02408542111515999, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09951798617839813, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09432473033666611, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019159704446792603, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1491709202528, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003691463964059949, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2772175073623657, "metrics/particle/momentum/all/accepted/pz_bias": 0.007268575020134449, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3573349714279175, "metrics/particle/momentum/all/accepted/samples": 28444.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3036501407623291, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0086605548858643, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3973.02294921875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3972.8828125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14463327825069427, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 743791.4375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01462872140109539, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7257742881774902, "metrics/particle/momentum/all/assigned/pz_bias": -0.014678925275802612, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.257523775100708, "metrics/particle/momentum/all/assigned/samples": 35068.0, "metrics/particle/momentum/all/efficiency": 0.7956586480140686, "metrics/particle/momentum/all/event/energy_bias": -0.26750728487968445, "metrics/particle/momentum/all/event/p3_error_mean": 20.065242767333984, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.507241249084473, "metrics/particle/momentum/all/event/px_bias": 0.24525651335716248, "metrics/particle/momentum/all/event/py_bias": 0.8516004085540771, "metrics/particle/momentum/all/event/pz_abs_mean": 14.790750503540039, "metrics/particle/momentum/all/event/pz_bias": 0.5938174724578857, "metrics/particle/momentum/all/event/pz_residual_rms": 23.65886878967285, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013980299234390259, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05914197862148285, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04893127456307411, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011841649189591408, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09962404519319534, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018537458963692188, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.18658222258090973, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0038158888928592205, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9272317290306091, "metrics/particle/momentum/class_1/accepted/samples": 15644.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12562024593353271, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3415565490722656, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4028374254703522, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.32978513836860657, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06600910425186157, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.638187885284424, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007937434129416943, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4379188120365143, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02989214099943638, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.492454767227173, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.9125590324401855, "metrics/particle/momentum/class_1/event/energy_bias": 3.344884157180786, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.256387710571289, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.583711624145508, "metrics/particle/momentum/class_1/event/px_bias": 0.7728481292724609, "metrics/particle/momentum/class_1/event/py_bias": 0.2468368262052536, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.58630084991455, "metrics/particle/momentum/class_1/event/pz_bias": -0.6853708624839783, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.260099411010742, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04072672501206398, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21723860502243042, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21608209609985352, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04883505403995514, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2552121579647064, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.056751467287540436, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.170295476913452, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3210758566856384, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.213112831115723, "metrics/particle/momentum/class_2/accepted/samples": 1022.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1206459999084473, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.899231493473053, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 53306.15234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 53305.58203125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.45464983582496643, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2724814.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09299655258655548, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.5821022987365723, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19753466546535492, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.760900020599365, "metrics/particle/momentum/class_2/assigned/samples": 2613.0, "metrics/particle/momentum/class_2/efficiency": 0.35363322496414185, "metrics/particle/momentum/class_2/event/energy_bias": -12.069622039794922, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.65561294555664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.366741180419922, "metrics/particle/momentum/class_2/event/px_bias": -0.48068681359291077, "metrics/particle/momentum/class_2/event/py_bias": 0.4119185209274292, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.6384916305542, "metrics/particle/momentum/class_2/event/pz_bias": -0.7268704771995544, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.153173446655273, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03690563514828682, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14557194709777832, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14683450758457184, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03526463359594345, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1887924075126648, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011504425201565027, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21933577954769135, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03835628926753998, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8927096128463745, "metrics/particle/momentum/class_3/accepted/samples": 11300.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3726341128349304, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7675929069519043, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9555963277816772, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8123246431350708, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18338516354560852, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.652677536010742, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008516966365277767, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.556324303150177, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.029847227036952972, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6629321575164795, "metrics/particle/momentum/class_3/assigned/samples": 14794.0, "metrics/particle/momentum/class_3/efficiency": 0.7439594268798828, "metrics/particle/momentum/class_3/event/energy_bias": 10.42696762084961, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.117532730102539, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.837388038635254, "metrics/particle/momentum/class_3/event/px_bias": -0.2734801471233368, "metrics/particle/momentum/class_3/event/py_bias": -0.09343357384204865, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.474597930908203, "metrics/particle/momentum/class_3/event/pz_bias": 1.4983878135681152, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.189579010009766, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017565418034791946, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10711096227169037, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1055307388305664, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.030700206756591797, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15745675563812256, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016025641933083534, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7162641286849976, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0737692341208458, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.59946346282959, "metrics/particle/momentum/class_4/accepted/samples": 312.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.13257859647274017, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.832566976547241, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.1955697536468506, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.11503529548645, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.09887005388736725, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.559995651245117, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01977401040494442, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8250394463539124, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.17830929160118103, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.991209030151367, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8813559412956238, "metrics/particle/momentum/class_4/event/energy_bias": -1.7759349346160889, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.4380226135253906, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.370203971862793, "metrics/particle/momentum/class_4/event/px_bias": 0.26745879650115967, "metrics/particle/momentum/class_4/event/py_bias": 0.18783029913902283, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.060293674468994, "metrics/particle/momentum/class_4/event/pz_bias": 0.3425352871417999, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.4682035446167, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013500809669494629, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03056052140891552, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027113040909171104, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009448816068470478, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06044184789061546, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.27874690294265747, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.11294568330049515, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6304835081100464, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04679407924413681, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.09991692006587982, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.09601621329784393, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05692175403237343, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6384733319282532, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.38030365109443665, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.05137629806995392, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.1299635171890259, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.19380125403404236, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.585369348526001, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.105126142501831, "metrics/particle/momentum/class_5/event/px_bias": -0.04088343679904938, "metrics/particle/momentum/class_5/event/py_bias": 0.09844832122325897, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9464362859725952, "metrics/particle/momentum/class_5/event/pz_bias": 0.1651359498500824, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.085964679718018, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.089390516281128, "step_valid_time": 1675.708991765976, "step_total_time": 1676.7983827590942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0084801639146221, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12940605882377276, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2850, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2827, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9353462422054479, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9341199606686332, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9347326992456543, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06588003933136677, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.927797833935018, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9265814487053425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.927189242374549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9353462422054479, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000781056494512, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13282099520041246, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2282, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2099, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9297629499561019, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9281332164767747, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9289473684210526, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07186678352322524, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9214223002633889, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9198071866783524, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9206140350877193, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9297629499561019, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005935814452271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1447980193655698, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1196, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9074355083459787, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9101978691019786, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9088145896656535, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0898021308980213, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8990895295902883, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9018264840182648, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9004559270516718, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9074355083459787} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_9000.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..1e8b1e5b7e0c4a6128f019009da8c6f62fad2d6c --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.4262978400909435, "train": {"Classification_binary": 3.338226079940796, "Classification": 0.024903155863285065, "Regression_pt": 0.17386414110660553, "Regression_eta": 0.009708795696496964, "Regression_sin_phi": 0.007263988722115755, "Regression_cos_phi": 0.008033011108636856, "Regression_energy": 0.17659443616867065, "Total": 9.913976669311523, "Total_unweighted": 3.7385936086066067}, "train_group": {}, "valid": {"Classification_binary": 3.475053151448568, "Classification": 0.024754765133063, "Regression_pt": 0.19029098749160767, "Regression_eta": 0.014418818056583405, "Regression_sin_phi": 0.008372418582439423, "Regression_cos_phi": 0.009001214678088823, "Regression_energy": 0.19670740763346353, "Total": 10.838088989257812, "Total_unweighted": 3.9185987630238137, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.024801021441817284, "diagnostic/loss/cld_pf/Classification_binary": 3.4724185466766357, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.008664185181260109, "diagnostic/loss/cld_pf/Regression_energy": 0.19562582671642303, "diagnostic/loss/cld_pf/Regression_eta": 0.013227615505456924, "diagnostic/loss/cld_pf/Regression_pt": 0.19049158692359924, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00771181657910347, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08087728172540665, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.22916483879089355, "diagnostic/regression/cld_pf/energy_residual_mean": 0.053758006542921066, "diagnostic/regression/cld_pf/energy_residual_rms": 0.47998273372650146, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04510168731212616, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.22313635051250458, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07226882129907608, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4747285842895508, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9153666496276855, "metrics/particle/class_1/pid_accuracy": 0.982182502746582, "metrics/particle/class_2/efficiency": 0.3252819776535034, "metrics/particle/class_2/pid_accuracy": 0.5824868679046631, "metrics/particle/class_3/efficiency": 0.7459564208984375, "metrics/particle/class_3/pid_accuracy": 0.9800259470939636, "metrics/particle/class_4/efficiency": 0.8728557229042053, "metrics/particle/class_4/pid_accuracy": 0.20462428033351898, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.40770789980888367, "metrics/particle/count/bias_mean": 1.9666666984558105, "metrics/particle/count/mae": 4.377124309539795, "metrics/particle/count/prediction_mean": 72.53202819824219, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.07321155816316605, "metrics/particle/event/energy_response_mean": 0.9555351138114929, "metrics/particle/event/met_abs_error": 7.155594348907471, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06195305287837982, "metrics/particle/event/scalar_pt_response_mean": 0.9679065942764282, "metrics/particle/event/vector_pt_closure": 0.06462319195270538, "metrics/particle/matched/delta_eta_abs_mean": 0.014544710516929626, "metrics/particle/matched/delta_phi_abs_mean": 0.014084153808653355, "metrics/particle/matched/delta_r_mean": 0.02264496684074402, "metrics/particle/matched/energy_relative_abs_mean": 0.1013319194316864, "metrics/particle/matched/pid_accuracy": 0.956924557685852, "metrics/particle/matched/pt_relative_abs_mean": 0.09318802505731583, "metrics/particle/matching/duplicate_fraction": 0.03734207898378372, "metrics/particle/matching/efficiency": 0.7958042025566101, "metrics/particle/matching/f1": 0.784866988658905, "metrics/particle/matching/purity": 0.7742263674736023, "metrics/particle/matching/target_coverage_dr0p05": 0.7418515086174011, "metrics/particle/matching/target_coverage_dr0p10": 0.837688148021698, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6803964376449585, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8403278589248657, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024228477850556374, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09958580881357193, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09386385232210159, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017969412729144096, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1482834815979004, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0053482744842767715, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4348301887512207, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.09658388048410416, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6719542741775513, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7853.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.35532885789871216, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.123447060585022, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.076395034790039, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9280971884727478, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1587476283311844, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.608323097229004, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018905671313405037, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9809525609016418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.12850871682167053, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5688316822052, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9997.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7746868133544922, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.457244396209717, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.09295654296875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.28998327255249, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.24258540570735931, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6230478882789612, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.769392967224121, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.3946712017059326, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.53100299835205, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.033227432519197464, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10889741778373718, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10656491667032242, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011439705267548561, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16018398106098175, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024032043293118477, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0993319749832153, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3141324520111084, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7290871143341064, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1498.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8845498561859131, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3165465593338013, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4092.06982421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4091.86474609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3086325526237488, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 209226.03125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08556149899959564, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0894522666931152, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5687660574913025, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.503560066223145, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2618.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5449254512786865, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.433204412460327, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.466632843017578, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.34024977684021, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04370933026075363, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08594074845314026, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.2773284912109375, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.0612244606018066, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.129972457885742, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02139209397137165, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10180868208408356, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09281683713197708, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014978216029703617, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14839528501033783, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007823501946404576, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13510854542255402, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.026020416989922523, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5816675424575806, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19173.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20909301936626434, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5841131210327148, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.47098278999328613, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3182321786880493, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11873561143875122, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.297426700592041, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004471400752663612, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34281134605407715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.032452259212732315, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7537047863006592, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22588.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8386038541793823, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -6.655492782592773, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.16329288482666, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.352206230163574, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.419116497039795, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.089781641960144, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.948774337768555, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5159995555877686, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.917607307434082, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02279454469680786, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.10156896710395813, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09382709860801697, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015615896321833134, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14900696277618408, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032604122534394264, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2682637870311737, "metrics/particle/momentum/all/accepted/pz_bias": -0.06057821959257126, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3142849206924438, "metrics/particle/momentum/all/accepted/samples": 28524.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3008541166782379, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7917440533638, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 304.9296569824219, "metrics/particle/momentum/all/assigned/pt_relative_bias": 304.7742614746094, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14422066509723663, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 57057.234375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014601028524339199, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.728295624256134, "metrics/particle/momentum/all/assigned/pz_bias": -0.09961544722318649, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3117918968200684, "metrics/particle/momentum/all/assigned/samples": 35203.0, "metrics/particle/momentum/all/efficiency": 0.7978964447975159, "metrics/particle/momentum/all/event/energy_bias": -12.545940399169922, "metrics/particle/momentum/all/event/p3_error_mean": 19.59484100341797, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.472177505493164, "metrics/particle/momentum/all/event/px_bias": -1.617992639541626, "metrics/particle/momentum/all/event/py_bias": 1.7987703084945679, "metrics/particle/momentum/all/event/pz_abs_mean": 14.243144989013672, "metrics/particle/momentum/all/event/pz_bias": -5.971895217895508, "metrics/particle/momentum/all/event/pz_residual_rms": 21.993494033813477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012904849834740162, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.06119967997074127, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04746469855308533, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002462995471432805, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09766076505184174, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001850551925599575, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17760342359542847, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.014981325715780258, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9593126177787781, "metrics/particle/momentum/class_1/accepted/samples": 15671.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1193709447979927, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.26587629318237305, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3214113414287567, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23769380152225494, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.062266573309898376, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.149252891540527, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007703081239014864, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4140021502971649, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0351366251707077, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5217792987823486, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9141340255737305, "metrics/particle/momentum/class_1/event/energy_bias": -1.5563071966171265, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.579224586486816, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.551936149597168, "metrics/particle/momentum/class_1/event/px_bias": -0.2060837596654892, "metrics/particle/momentum/class_1/event/py_bias": 0.2801807224750519, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.986159324645996, "metrics/particle/momentum/class_1/event/pz_bias": -1.1817245483398438, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.5986270904541, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.042480237782001495, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21856637299060822, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21985936164855957, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06262117624282837, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2576242983341217, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04906054213643074, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.2024879455566406, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.6089123487472534, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.150690078735352, "metrics/particle/momentum/class_2/accepted/samples": 958.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1807231903076172, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8373968005180359, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3997.297607421875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3996.626708984375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.48338931798934937, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 206830.3125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09406495094299316, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.8264007568359375, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.6235209703445435, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.361083984375, "metrics/particle/momentum/class_2/assigned/samples": 2679.0, "metrics/particle/momentum/class_2/efficiency": 0.33148789405822754, "metrics/particle/momentum/class_2/event/energy_bias": -15.564213752746582, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.347118377685547, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.623559951782227, "metrics/particle/momentum/class_2/event/px_bias": -0.612536609172821, "metrics/particle/momentum/class_2/event/py_bias": 0.8349025845527649, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.211478233337402, "metrics/particle/momentum/class_2/event/pz_bias": -2.463742256164551, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.134319305419922, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03503166139125824, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1483452469110489, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1477278620004654, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0401616171002388, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19014635682106018, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013138302601873875, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22414076328277588, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.08106299489736557, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8304157257080078, "metrics/particle/momentum/class_3/accepted/samples": 11417.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3567092716693878, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3808916807174683, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3248461484909058, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.176648736000061, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18012380599975586, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.0490083694458, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008343426510691643, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5372729301452637, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.08582217991352081, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.417809009552002, "metrics/particle/momentum/class_3/assigned/samples": 14862.0, "metrics/particle/momentum/class_3/efficiency": 0.7516623735427856, "metrics/particle/momentum/class_3/event/energy_bias": 7.301932334899902, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.936984062194824, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.577870845794678, "metrics/particle/momentum/class_3/event/px_bias": -0.6247246861457825, "metrics/particle/momentum/class_3/event/py_bias": 0.5271008610725403, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.524528503417969, "metrics/particle/momentum/class_3/event/pz_bias": -2.682661533355713, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.512381553649902, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017122548073530197, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09612995386123657, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.09806106239557266, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.02204994112253189, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1469295471906662, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006451612804085016, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5315361022949219, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030745863914489746, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.205275774002075, "metrics/particle/momentum/class_4/accepted/samples": 310.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.21618087589740753, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5303159952163696, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3405277729034424, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2544900178909302, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.09348441660404205, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.452837944030762, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014164306223392487, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7267340421676636, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12552207708358765, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.8018269538879395, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8757061958312988, "metrics/particle/momentum/class_4/event/energy_bias": -1.9438992738723755, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0122315883636475, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.176264524459839, "metrics/particle/momentum/class_4/event/px_bias": -0.19867539405822754, "metrics/particle/momentum/class_4/event/py_bias": 0.17736569046974182, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6670156717300415, "metrics/particle/momentum/class_4/event/pz_bias": 0.23514023423194885, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.101176738739014, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011899369768798351, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03124210610985756, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02899867668747902, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00869440846145153, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06867427378892899, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2080899178981781, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03656088188290596, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5618972182273865, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.026270246133208275, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05381089821457863, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05203097313642502, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013839288614690304, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15648232400417328, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.29731228947639465, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.01773916929960251, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0668715238571167, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": -0.7834540009498596, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1711769104003906, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9085104465484619, "metrics/particle/momentum/class_5/event/px_bias": 0.024027826264500618, "metrics/particle/momentum/class_5/event/py_bias": -0.0207795649766922, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5916236639022827, "metrics/particle/momentum/class_5/event/pz_bias": 0.12109297513961792, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.3036012649536133, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.1916909217834473, "step_valid_time": 1864.9907586574554, "step_total_time": 1866.1824505329132, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9871593626787223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12584680023694061, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2859, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9382999671808336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9404605263157895, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9393790044356826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059539473684210524, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9343616672136528, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9365131578947369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354361754558896, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9382999671808336, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9818331655470575, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1354253523231953, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2262, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2112, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2100, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9271290605794557, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9336870026525199, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9303964757709251, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06631299734748011, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9218612818261633, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9283819628647215, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9251101321585903, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9271290605794557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9828457427804858, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14843874656304412, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1200, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9104704097116844, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9153318077803204, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9128946367440092, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08466819221967964, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9066767830045523, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9115179252479023, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9090909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9104704097116844} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/hyperparameters.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..f4309183f97eb7f8df3c30960db166b4c974084f --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3114258, "dataset": "cld", "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": true, "use_flash_attn_varlen": true, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 128, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf", "cld_edm_ww_fullhad_pf", "cld_edm_qq_pf"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/model_kwargs.pkl b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4eea1862dbebc3c3d6c6c45dc08cfb5ff458fb19 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2477072065a5878ead61a8d52902e829e79b4bddf244254c1319529f621c25d +size 4978 diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/particleflow_spec.yaml b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e16807546e33fc7d7617d20c3f827e72313e4e07 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..769d6da69e88b739d15d8cc41f62417e51684e24 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_eta.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2afa61c618745be9a149c54cad15d80ed48f7132 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..eed02a8e1cf0aeaa1624437e79df450a831d04b9 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..236b0fc3cdaff43cd1a7412972350e56a8a9c72d Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..bf362fcfe69d68e5e70913538e645f236d0cdf8e Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6258fc7103c8d420523f7ce516e8ee7a76c5eff5 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..45ce2fc99b40eb95d2d0120bc695538cce26f899 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..83d8facbe46ec34060657c2c61367219ef7ef600 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..8d6650e8a17763973f1442ad5e45b2c952e3bc4f Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dcf3cb32c4f9079a7ff0815d5913165995983602 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..ba56b74f421099b14e7ac5dd0135dfe376e869f1 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_qq_pf/num_elements.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..01c3f9bd923c265413734cc3a4b4f1375facf9b1 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..123da80e61306a26230591702a169196fc38d901 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bf923a846a6ab2901eac2058358483124c165cd4 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..2479f5fe9e9b12f3d2b6e2318cde8c4bcc42e728 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..84db3d27943c2948fb47e3fa6ca482eca37bcf33 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..87a7cea8b41c08fa31f00198eb89f9c37245f707 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f1188381db75a892111b11779ae5cdfc338f1cc1 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..9c56e17457129f98e54650fd75683d7a57a389da Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9fb2da65f1d974b2b8f3445f39ae42ad9cd1a06f Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..dd7e7433c0f381c988d3c37a1e9c6f30c6120f86 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..051e39b27c69048bda97d1049a634a15cf82b6a9 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..a0eff4c2eee3c8854cdd1971afc10246e6d543fc Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d5e5eaa33afb47faf911609f20f1fb6a846ae373 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..5863cd266cfc178b91a8e99c34ec43ad0bc8e75a Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7c9ce0c42fdabd85c06f5b5a2657fdc9afd80e68 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..37f98ebdc0e937b1e2395080d7a78b3568a0f7fe Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7566dc4ff420ebd51f30ddb6abcb1af43b566323 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..800e85fa94d5a6e46c830720e9e020ad418ce1c2 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..da7e546f7e362ed20304296ed089ebab742d7326 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..e7b619c12852175c310739da0ab035ab3ed2c359 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..76535c70601b5e9d6134b0774e9307a86498a4c4 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..ed1ed15910abe57e853aac68dcc1a2280d8cdcc8 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ddb8357aa4bd8ec5cb205c4c3408a82cd3234ed8 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..4836501e18e89599703fe6544f810383185d576c Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/runs/train/events.out.tfevents.1793178753.nid005546.24715.0 b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/runs/train/events.out.tfevents.1793178753.nid005546.24715.0 new file mode 100644 index 0000000000000000000000000000000000000000..d3fe23fe7b599e8a571980df2ffea3fb8f0def45 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/runs/train/events.out.tfevents.1793178753.nid005546.24715.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceef311fd91157f1393962913b350425e25bac550671f943ceac7b531af759ec +size 15743402 diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/runs/valid/events.out.tfevents.1793178753.nid005546.24715.1 b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/runs/valid/events.out.tfevents.1793178753.nid005546.24715.1 new file mode 100644 index 0000000000000000000000000000000000000000..e33b51fbac3966dd0cc743626e12717783ecab77 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/runs/valid/events.out.tfevents.1793178753.nid005546.24715.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc73b83d71bac222dc3d0965e2664b5038abd24ddd2a47950ca9548f1d8ee714 +size 28914590 diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/scenario-manifest.json b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..33281b4d5ee188a908e1c00b23c952585c8aa36b --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/scenario-manifest.json @@ -0,0 +1,818 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239270-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239270-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "cld_pf_attention", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 128, + "gpu_batch_multiplier": 128, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "seed": 12345, + "gpu_batch_multiplier": 128 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": true, + "use_flash_attn_varlen": true, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 128, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf", + "cld_edm_ww_fullhad_pf", + "cld_edm_qq_pf" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 1 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--seed", + "12345", + "--gpu_batch_multiplier", + "128" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" +} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train-config.yaml b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78340e3c850f87401d52828053d182ad70de6120 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train-config.yaml @@ -0,0 +1,278 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +- cld_edm_ww_fullhad_pf +- cld_edm_qq_pf +gpu_batch_multiplier: 128 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: true + use_jagged_attention: true + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: false + trainable: all + type: attention +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + cld_edm_qq_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ww_fullhad_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train.log b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train.log new file mode 100644 index 0000000000000000000000000000000000000000..502e291991bd11c1a064147f6e9d70934de246b7 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train.log @@ -0,0 +1,13849 @@ +rank0 [2026-10-28 11:12:24,870] INFO : Will use single-gpu: AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,871] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-10-28 11:12:24,877] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,877] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,878] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,878] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,878] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,915] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,915] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,915] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,915] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,915] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,918] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,918] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,921] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,921] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,924] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,924] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,927] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:24,927] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,929] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:24,929] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,932] INFO : Backbone initialization took 0.02s +rank0 [2026-10-28 11:12:24,932] INFO : backbone parameters: 2374656 +rank0 [2026-10-28 11:12:24,933] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:24,937] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:24,937] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:24,937] INFO : jagged_attention enabled=True +rank0 [2026-10-28 11:12:24,938] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-28 11:12:24,938] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:24,938] INFO : classification_norm parameters: 512 +rank0 [2026-10-28 11:12:24,938] INFO : regression_norm parameters: 512 +rank0 [2026-10-28 11:12:24,938] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:24,938] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:24,938] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:24,939] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:24,939] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:24,939] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,939] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,939] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:24,939] INFO : Total MLPF parameters: 3114258 +rank0 [2026-10-28 11:12:24,939] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:24,940] INFO : Instantiated model +rank0 [2026-10-28 11:12:24,940] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,313] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,314] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,394] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +131 nn_energy.nn.1.0.bias 256 0 +132 nn_energy.nn.1.2.weight 256 0 +133 nn_energy.nn.1.2.bias 256 0 +134 nn_energy.nn.1.4.weight 256 0 +135 nn_energy.nn.1.4.bias 1 0 + +[136 rows x 3 columns] +rank0 [2026-10-28 11:12:25,394] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,396] INFO : Trainable parameters: 3114258 +rank0 [2026-10-28 11:12:25,396] INFO : Non-trainable parameters: 0 +rank0 [2026-10-28 11:12:25,396] INFO : Total parameters: 3114258 +rank0 [2026-10-28 11:12:25,398] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 +backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 +backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 +backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 +backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 +backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 +backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,401] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206 +rank0 [2026-10-28 11:12:25,401] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206 +rank0 [2026-10-28 11:12:25,401] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,401] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,271] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,312] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,343] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,376] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,402] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,441] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,464] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,490] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,515] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,541] INFO : train_dataset: cld_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,541] INFO : sample=cld_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,561] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,596] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,647] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,694] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,733] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,762] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,789] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,815] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,839] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,865] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,865] INFO : sample=cld_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,898] INFO : train_dataset: cld_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:32,922] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,966] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,994] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,017] INFO : train_dataset: cld_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,053] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,080] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,122] INFO : train_dataset: cld_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,148] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset: cld_edm_qq_pf, 90598 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,177] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,177] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,177] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,183] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,189] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,195] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,200] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,205] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,211] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,216] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,222] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,227] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,233] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,233] INFO : sample=cld_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,238] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,243] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,248] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,253] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,258] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,264] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,269] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,274] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,279] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,284] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,284] INFO : sample=cld_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,289] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,294] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,300] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,305] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,310] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,315] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,320] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,325] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,330] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,335] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,336] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,336] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,370] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,373] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,374] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-28 11:12:33,374] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,374] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,380] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,380] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,397] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,640] INFO : Created train_iterator +rank0 [2026-10-28 11:13:37,530] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.284732975959778, 'Classification': 0.140388917773962, 'Regression_pt': 0.4447642934322357, 'Regression_eta': 0.11849542165175081, 'Regression_sin_phi': 0.055837437864393, 'Regression_cos_phi': 0.08143645193427801, 'Regression_energy': 0.4833901843428612}, weights={'Classification_binary': 1.0, 'Classification': 37.64352035522461, 'Regression_pt': 11.882098197937012, 'Regression_eta': 44.59862518310547, 'Regression_sin_phi': 94.64497375488281, 'Regression_cos_phi': 64.89395141601562, 'Regression_energy': 10.93264389038086} +rank0 [2026-10-28 11:13:39,439] INFO : Step 100/50000 rank0 | Train Loss: 5.7439 | Train Loss Unweighted: 5.7439 | Train Loss Components: Classification: 0.0486 | Classification_binary: 4.8688 | Regression_cos_phi: 0.0220 | Regression_energy: 0.3793 | Regression_eta: 0.0353 | Regression_pt: 0.3669 | Regression_sin_phi: 0.0231 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 4.8688 | Regression_cos_phi: 0.0220 | Regression_energy: 0.3793 | Regression_eta: 0.0353 | Regression_pt: 0.3669 | Regression_sin_phi: 0.0231 | LR: 1.00e-03 | DataLoad Time: 0.0553s | Model Forward Time: 2.2113s | Phase Times: max_pre_step_barrier: 0.0005 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.1093 | max_backward: 0.0335 | max_optimizer_step: 0.1019 | max_tensorboard_logging: 1.2837 | max_tensorboard_flush: 0.0698 | Memory: rss_MB: 2689.1 | uss_MB: 1984.0 | child_rss_MB: 3276.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:13:39,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 97.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:02,256] INFO : Step 200/50000 rank0 | Train Loss: 18.4973 | Train Loss Unweighted: 5.5715 | Train Loss Components: Classification: 0.0446 | Classification_binary: 4.7553 | Regression_cos_phi: 0.0150 | Regression_energy: 0.3632 | Regression_eta: 0.0322 | Regression_pt: 0.3443 | Regression_sin_phi: 0.0168 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.6797 | Classification_binary: 4.7553 | Regression_cos_phi: 0.9765 | Regression_energy: 3.9711 | Regression_eta: 1.4367 | Regression_pt: 4.0912 | Regression_sin_phi: 1.5868 | LR: 1.00e-03 | DataLoad Time: 0.0015s | Model Forward Time: 0.9800s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0139 | max_loss: 0.0030 | max_backward: 0.0331 | max_optimizer_step: 0.0991 | max_tensorboard_logging: 0.2484 | max_tensorboard_flush: 0.0083 | Memory: rss_MB: 2688.7 | uss_MB: 1985.5 | child_rss_MB: 3280.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:14:02,468] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 105.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:23,115] INFO : Step 300/50000 rank0 | Train Loss: 18.2072 | Train Loss Unweighted: 5.6801 | Train Loss Components: Classification: 0.0399 | Classification_binary: 4.8508 | Regression_cos_phi: 0.0148 | Regression_energy: 0.3696 | Regression_eta: 0.0277 | Regression_pt: 0.3635 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5038 | Classification_binary: 4.8508 | Regression_cos_phi: 0.9608 | Regression_energy: 4.0408 | Regression_eta: 1.2365 | Regression_pt: 4.3193 | Regression_sin_phi: 1.2954 | LR: 1.00e-03 | DataLoad Time: 0.0019s | Model Forward Time: 1.1496s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0833 | max_forward: 0.0132 | max_loss: 0.0033 | max_backward: 0.0317 | max_optimizer_step: 0.0966 | max_tensorboard_logging: 0.3505 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 2688.4 | uss_MB: 1986.2 | child_rss_MB: 3276.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:14:23,325] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 105.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:43,387] INFO : Step 400/50000 rank0 | Train Loss: 16.8231 | Train Loss Unweighted: 5.3455 | Train Loss Components: Classification: 0.0407 | Classification_binary: 4.5716 | Regression_cos_phi: 0.0122 | Regression_energy: 0.3465 | Regression_eta: 0.0180 | Regression_pt: 0.3432 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5309 | Classification_binary: 4.5716 | Regression_cos_phi: 0.7946 | Regression_energy: 3.7878 | Regression_eta: 0.8007 | Regression_pt: 4.0779 | Regression_sin_phi: 1.2596 | LR: 1.00e-03 | DataLoad Time: 0.0514s | Model Forward Time: 0.9896s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 2688.1 | uss_MB: 1986.7 | child_rss_MB: 3276.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:14:43,599] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:03,132] INFO : Step 500/50000 rank0 | Train Loss: 17.1555 | Train Loss Unweighted: 5.3923 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.6286 | Regression_cos_phi: 0.0142 | Regression_energy: 0.3379 | Regression_eta: 0.0253 | Regression_pt: 0.3321 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5174 | Classification_binary: 4.6286 | Regression_cos_phi: 0.9193 | Regression_energy: 3.6937 | Regression_eta: 1.1282 | Regression_pt: 3.9459 | Regression_sin_phi: 1.3223 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.0920s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0159 | max_loss: 0.0030 | max_backward: 0.0381 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.2593 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 2690.0 | uss_MB: 1989.0 | child_rss_MB: 3277.4 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:15:03,350] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:22,123] INFO : Step 600/50000 rank0 | Train Loss: 17.4910 | Train Loss Unweighted: 5.3451 | Train Loss Components: Classification: 0.0374 | Classification_binary: 4.5458 | Regression_cos_phi: 0.0137 | Regression_energy: 0.3606 | Regression_eta: 0.0381 | Regression_pt: 0.3392 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4074 | Classification_binary: 4.5458 | Regression_cos_phi: 0.8911 | Regression_energy: 3.9423 | Regression_eta: 1.6994 | Regression_pt: 4.0307 | Regression_sin_phi: 0.9744 | LR: 1.00e-03 | DataLoad Time: 0.0016s | Model Forward Time: 1.0824s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0163 | max_loss: 0.0030 | max_backward: 0.1418 | max_optimizer_step: 0.0988 | max_tensorboard_logging: 0.2480 | max_tensorboard_flush: 0.0079 | Memory: rss_MB: 2623.3 | uss_MB: 1921.6 | child_rss_MB: 3274.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:15:22,352] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:41,530] INFO : Step 700/50000 rank0 | Train Loss: 16.5769 | Train Loss Unweighted: 5.3617 | Train Loss Components: Classification: 0.0386 | Classification_binary: 4.6112 | Regression_cos_phi: 0.0115 | Regression_energy: 0.3339 | Regression_eta: 0.0194 | Regression_pt: 0.3335 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4522 | Classification_binary: 4.6112 | Regression_cos_phi: 0.7476 | Regression_energy: 3.6505 | Regression_eta: 0.8641 | Regression_pt: 3.9629 | Regression_sin_phi: 1.2884 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.0783s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0029 | max_backward: 0.0380 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.2494 | max_tensorboard_flush: 0.0079 | Memory: rss_MB: 2627.4 | uss_MB: 1927.1 | child_rss_MB: 3266.4 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:15:41,746] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:00,427] INFO : Step 800/50000 rank0 | Train Loss: 16.1142 | Train Loss Unweighted: 5.2348 | Train Loss Components: Classification: 0.0371 | Classification_binary: 4.5277 | Regression_cos_phi: 0.0125 | Regression_energy: 0.3125 | Regression_eta: 0.0205 | Regression_pt: 0.3100 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3981 | Classification_binary: 4.5277 | Regression_cos_phi: 0.8137 | Regression_energy: 3.4167 | Regression_eta: 0.9155 | Regression_pt: 3.6837 | Regression_sin_phi: 1.3589 | LR: 9.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0135 | max_loss: 0.0029 | max_backward: 0.0330 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.2715 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 2626.2 | uss_MB: 1926.3 | child_rss_MB: 3272.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:16:00,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:19,403] INFO : Step 900/50000 rank0 | Train Loss: 15.1225 | Train Loss Unweighted: 5.2828 | Train Loss Components: Classification: 0.0363 | Classification_binary: 4.6219 | Regression_cos_phi: 0.0111 | Regression_energy: 0.2949 | Regression_eta: 0.0171 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3654 | Classification_binary: 4.6219 | Regression_cos_phi: 0.7179 | Regression_energy: 3.2241 | Regression_eta: 0.7610 | Regression_pt: 3.4620 | Regression_sin_phi: 0.9702 | LR: 9.99e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1217s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0158 | max_loss: 0.0031 | max_backward: 0.0346 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.2938 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 2628.9 | uss_MB: 1928.9 | child_rss_MB: 3267.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:16:19,626] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:38,580] INFO : Step 1000/50000 rank0 | Train Loss: 16.1139 | Train Loss Unweighted: 5.2604 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.5681 | Regression_cos_phi: 0.0140 | Regression_energy: 0.3061 | Regression_eta: 0.0219 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5186 | Classification_binary: 4.5681 | Regression_cos_phi: 0.9086 | Regression_energy: 3.3466 | Regression_eta: 0.9780 | Regression_pt: 3.5238 | Regression_sin_phi: 1.2702 | LR: 9.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0038s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0131 | max_loss: 0.0029 | max_backward: 0.0330 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 2629.0 | uss_MB: 1929.7 | child_rss_MB: 3266.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:16:38,794] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:38,902] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:16:38,990] INFO : Running validation on rank0 +rank0 [2026-10-28 11:16:38,990] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:16:38,990] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:16:40,188] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:16:40,374] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:16:40,471] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:16:40,532] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:16:40,591] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:16:40,642] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:16:40,701] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:16:40,763] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:16:40,820] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:16:40,907] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:16:40,995] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:16:41,054] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:16:41,114] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:16:41,164] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:16:41,224] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:16:41,285] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:16:41,343] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:16:41,347] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:16:41,350] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:16:41,353] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:16:41,353] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:16:41,356] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:16:41,359] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:16:41,359] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:16:41,362] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:16:41,365] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:16:41,365] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:16:41,368] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:16:41,370] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:16:41,371] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:16:41,374] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:16:41,377] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:16:47,276] INFO : VALIDATION | Step=1000/50000 | Train Loss=16.1139 | Valid Loss=15.5723 | Train Loss Unweighted=5.2604 | Valid Loss Unweighted=5.2125 | Stale=0 | ETA=207.3m +rank0 [2026-10-28 11:16:47,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:47,346] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:16:47,356] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:16:47,489] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:49,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:49,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:50,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:51,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:51,239] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:16:51,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:51,297] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:16:51,300] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:16:51,367] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:52,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:53,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:53,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:54,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:54,596] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:54,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:54,651] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:16:54,654] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:16:54,722] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:56,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:56,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:57,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:57,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:57,712] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:57,721] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:16:57,978] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:16:58,882] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:00,379] INFO : Plotted jets +rank0 [2026-10-28 11:17:01,461] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:01,494] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:01,721] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:02,111] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:03,497] INFO : Plotted jets +rank0 [2026-10-28 11:17:04,598] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:04,634] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:17:05,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:17:05,489] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:06,850] INFO : Plotted jets +rank0 [2026-10-28 11:17:07,886] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:26,618] INFO : Step 1100/50000 rank0 | Train Loss: 15.8770 | Train Loss Unweighted: 5.1017 | Train Loss Components: Classification: 0.0383 | Classification_binary: 4.4144 | Regression_cos_phi: 0.0126 | Regression_energy: 0.3013 | Regression_eta: 0.0295 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4406 | Classification_binary: 4.4144 | Regression_cos_phi: 0.8204 | Regression_energy: 3.2940 | Regression_eta: 1.3138 | Regression_pt: 3.4940 | Regression_sin_phi: 1.0997 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.0090s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0162 | max_loss: 0.0030 | max_backward: 0.0340 | max_optimizer_step: 0.0976 | max_tensorboard_logging: 0.2568 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 2894.5 | uss_MB: 2380.7 | child_rss_MB: 3264.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:17:26,832] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:45,456] INFO : Step 1200/50000 rank0 | Train Loss: 15.2241 | Train Loss Unweighted: 5.0294 | Train Loss Components: Classification: 0.0381 | Classification_binary: 4.3872 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2798 | Regression_eta: 0.0218 | Regression_pt: 0.2764 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4332 | Classification_binary: 4.3872 | Regression_cos_phi: 0.8240 | Regression_energy: 3.0593 | Regression_eta: 0.9709 | Regression_pt: 3.2842 | Regression_sin_phi: 1.2653 | LR: 9.99e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.0210s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0145 | max_loss: 0.0031 | max_backward: 0.0334 | max_optimizer_step: 0.1016 | max_tensorboard_logging: 0.2635 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2895.4 | uss_MB: 2381.5 | child_rss_MB: 3260.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:17:45,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:03,944] INFO : Step 1300/50000 rank0 | Train Loss: 16.5995 | Train Loss Unweighted: 5.1553 | Train Loss Components: Classification: 0.0379 | Classification_binary: 4.4367 | Regression_cos_phi: 0.0140 | Regression_energy: 0.3180 | Regression_eta: 0.0358 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4248 | Classification_binary: 4.4367 | Regression_cos_phi: 0.9105 | Regression_energy: 3.4767 | Regression_eta: 1.5985 | Regression_pt: 3.5686 | Regression_sin_phi: 1.1836 | LR: 9.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.0529s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0173 | max_loss: 0.0458 | max_backward: 0.0381 | max_optimizer_step: 0.0987 | max_tensorboard_logging: 0.2586 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2897.8 | uss_MB: 2384.4 | child_rss_MB: 3259.3 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:18:04,158] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:22,241] INFO : Step 1400/50000 rank0 | Train Loss: 15.6765 | Train Loss Unweighted: 5.0366 | Train Loss Components: Classification: 0.0354 | Classification_binary: 4.3577 | Regression_cos_phi: 0.0103 | Regression_energy: 0.3028 | Regression_eta: 0.0285 | Regression_pt: 0.2879 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3338 | Classification_binary: 4.3577 | Regression_cos_phi: 0.6711 | Regression_energy: 3.3106 | Regression_eta: 1.2729 | Regression_pt: 3.4213 | Regression_sin_phi: 1.3093 | LR: 9.98e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.1322s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0541 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1000 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 2895.2 | uss_MB: 2381.7 | child_rss_MB: 3263.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:22,452] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:40,629] INFO : Step 1500/50000 rank0 | Train Loss: 14.8835 | Train Loss Unweighted: 5.0770 | Train Loss Components: Classification: 0.0360 | Classification_binary: 4.4380 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2851 | Regression_eta: 0.0205 | Regression_pt: 0.2749 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3537 | Classification_binary: 4.4380 | Regression_cos_phi: 0.7368 | Regression_energy: 3.1164 | Regression_eta: 0.9129 | Regression_pt: 3.2668 | Regression_sin_phi: 1.0588 | LR: 9.98e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.0852s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0031 | max_backward: 0.0327 | max_optimizer_step: 0.1007 | max_tensorboard_logging: 0.3089 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 2896.2 | uss_MB: 2383.5 | child_rss_MB: 3262.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:18:40,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:59,087] INFO : Step 1600/50000 rank0 | Train Loss: 15.6863 | Train Loss Unweighted: 5.0281 | Train Loss Components: Classification: 0.0352 | Classification_binary: 4.3291 | Regression_cos_phi: 0.0130 | Regression_energy: 0.3119 | Regression_eta: 0.0229 | Regression_pt: 0.3037 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3262 | Classification_binary: 4.3291 | Regression_cos_phi: 0.8432 | Regression_energy: 3.4103 | Regression_eta: 1.0220 | Regression_pt: 3.6090 | Regression_sin_phi: 1.1465 | LR: 9.98e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1070s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0030 | max_backward: 0.1295 | max_optimizer_step: 0.1032 | max_tensorboard_logging: 0.2557 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2792.1 | uss_MB: 2278.7 | child_rss_MB: 3258.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:59,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:17,632] INFO : Step 1700/50000 rank0 | Train Loss: 14.0979 | Train Loss Unweighted: 4.9267 | Train Loss Components: Classification: 0.0373 | Classification_binary: 4.3204 | Regression_cos_phi: 0.0094 | Regression_energy: 0.2711 | Regression_eta: 0.0208 | Regression_pt: 0.2594 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4023 | Classification_binary: 4.3204 | Regression_cos_phi: 0.6120 | Regression_energy: 2.9634 | Regression_eta: 0.9255 | Regression_pt: 3.0828 | Regression_sin_phi: 0.7915 | LR: 9.97e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.9851s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0030 | max_backward: 0.0331 | max_optimizer_step: 0.1006 | max_tensorboard_logging: 0.2450 | max_tensorboard_flush: 0.0080 | Memory: rss_MB: 2794.1 | uss_MB: 2281.4 | child_rss_MB: 3263.9 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:19:17,845] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:36,529] INFO : Step 1800/50000 rank0 | Train Loss: 15.1663 | Train Loss Unweighted: 5.0728 | Train Loss Components: Classification: 0.0393 | Classification_binary: 4.4279 | Regression_cos_phi: 0.0121 | Regression_energy: 0.2843 | Regression_eta: 0.0230 | Regression_pt: 0.2748 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4806 | Classification_binary: 4.4279 | Regression_cos_phi: 0.7833 | Regression_energy: 3.1083 | Regression_eta: 1.0265 | Regression_pt: 3.2651 | Regression_sin_phi: 1.0746 | LR: 9.97e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1981s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0597 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1064 | max_tensorboard_logging: 0.3634 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2794.1 | uss_MB: 2281.4 | child_rss_MB: 3257.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:19:36,740] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:55,114] INFO : Step 1900/50000 rank0 | Train Loss: 14.0118 | Train Loss Unweighted: 4.9192 | Train Loss Components: Classification: 0.0345 | Classification_binary: 4.3004 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2760 | Regression_eta: 0.0131 | Regression_pt: 0.2762 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2986 | Classification_binary: 4.3004 | Regression_cos_phi: 0.5986 | Regression_energy: 3.0173 | Regression_eta: 0.5823 | Regression_pt: 3.2816 | Regression_sin_phi: 0.9329 | LR: 9.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.0819s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0166 | max_loss: 0.0028 | max_backward: 0.0383 | max_optimizer_step: 0.1905 | max_tensorboard_logging: 0.2514 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2797.7 | uss_MB: 2286.1 | child_rss_MB: 3253.2 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:19:55,328] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:13,387] INFO : Step 2000/50000 rank0 | Train Loss: 15.0794 | Train Loss Unweighted: 5.0277 | Train Loss Components: Classification: 0.0378 | Classification_binary: 4.3431 | Regression_cos_phi: 0.0113 | Regression_energy: 0.3091 | Regression_eta: 0.0165 | Regression_pt: 0.3004 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4222 | Classification_binary: 4.3431 | Regression_cos_phi: 0.7329 | Regression_energy: 3.3791 | Regression_eta: 0.7370 | Regression_pt: 3.5700 | Regression_sin_phi: 0.8951 | LR: 9.96e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.0935s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0136 | max_loss: 0.0031 | max_backward: 0.1263 | max_optimizer_step: 0.1027 | max_tensorboard_logging: 0.2513 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2796.1 | uss_MB: 2283.9 | child_rss_MB: 3256.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:20:13,614] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:13,631] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:20:13,713] INFO : Running validation on rank0 +rank0 [2026-10-28 11:20:13,713] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:20:13,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:20:14,899] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:20:15,041] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:20:15,172] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:20:15,244] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:20:15,312] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:20:15,378] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:20:15,446] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:20:15,515] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:20:15,615] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:20:15,735] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:20:15,856] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:20:15,927] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:20:15,995] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:20:16,062] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:20:16,130] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:20:16,199] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:20:16,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:20:16,312] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:20:16,315] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:20:16,318] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:20:16,319] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:20:16,321] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:20:16,324] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:20:16,325] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:20:16,327] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:20:16,330] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:20:16,333] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:20:16,335] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:20:16,337] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:20:16,339] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:20:16,342] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:20:16,344] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:20:22,186] INFO : VALIDATION | Step=2000/50000 | Train Loss=15.0794 | Valid Loss=14.4329 | Train Loss Unweighted=5.0277 | Valid Loss Unweighted=4.9395 | Stale=0 | ETA=187.5m +rank0 [2026-10-28 11:20:22,195] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:22,255] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:20:22,259] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:22,403] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:23,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:24,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:25,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:25,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:25,816] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:25,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:25,873] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:25,876] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:25,941] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:27,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:27,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:28,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:29,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:29,152] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:29,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:29,210] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:29,213] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:20:29,286] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:30,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:31,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:31,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:32,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:32,270] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:32,277] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:32,539] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:32,941] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:34,380] INFO : Plotted jets +rank0 [2026-10-28 11:20:35,460] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:35,493] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:35,722] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:36,496] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:37,912] INFO : Plotted jets +rank0 [2026-10-28 11:20:39,003] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:39,036] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:39,239] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:39,597] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:40,965] INFO : Plotted jets +rank0 [2026-10-28 11:20:42,009] INFO : Plotted jet ratio +rank0 [2026-10-28 11:21:00,513] INFO : Step 2100/50000 rank0 | Train Loss: 15.2360 | Train Loss Unweighted: 5.0142 | Train Loss Components: Classification: 0.0377 | Classification_binary: 4.3463 | Regression_cos_phi: 0.0125 | Regression_energy: 0.3000 | Regression_eta: 0.0163 | Regression_pt: 0.2884 | Regression_sin_phi: 0.0130 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4184 | Classification_binary: 4.3463 | Regression_cos_phi: 0.8085 | Regression_energy: 3.2801 | Regression_eta: 0.7280 | Regression_pt: 3.4265 | Regression_sin_phi: 1.2281 | LR: 9.96e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.0428s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.1015 | max_tensorboard_logging: 0.2759 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3018.4 | uss_MB: 2505.5 | child_rss_MB: 3244.0 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:21:00,727] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:18,664] INFO : Step 2200/50000 rank0 | Train Loss: 13.5599 | Train Loss Unweighted: 4.8168 | Train Loss Components: Classification: 0.0383 | Classification_binary: 4.2581 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2467 | Regression_eta: 0.0173 | Regression_pt: 0.2362 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4428 | Classification_binary: 4.2581 | Regression_cos_phi: 0.7080 | Regression_energy: 2.6966 | Regression_eta: 0.7699 | Regression_pt: 2.8071 | Regression_sin_phi: 0.8774 | LR: 9.96e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.0685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0160 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1039 | max_tensorboard_logging: 0.2707 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3018.9 | uss_MB: 2508.1 | child_rss_MB: 3241.4 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:21:18,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:36,726] INFO : Step 2300/50000 rank0 | Train Loss: 13.7804 | Train Loss Unweighted: 4.8773 | Train Loss Components: Classification: 0.0335 | Classification_binary: 4.2668 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2773 | Regression_eta: 0.0172 | Regression_pt: 0.2662 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2629 | Classification_binary: 4.2668 | Regression_cos_phi: 0.5526 | Regression_energy: 3.0311 | Regression_eta: 0.7682 | Regression_pt: 3.1631 | Regression_sin_phi: 0.7357 | LR: 9.95e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1982s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0879 | max_forward: 0.0150 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.0999 | max_tensorboard_logging: 0.3581 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 3019.4 | uss_MB: 2506.5 | child_rss_MB: 3238.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:21:36,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:54,814] INFO : Step 2400/50000 rank0 | Train Loss: 14.2686 | Train Loss Unweighted: 4.8262 | Train Loss Components: Classification: 0.0322 | Classification_binary: 4.2091 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2748 | Regression_eta: 0.0159 | Regression_pt: 0.2703 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2106 | Classification_binary: 4.2091 | Regression_cos_phi: 0.7286 | Regression_energy: 3.0047 | Regression_eta: 0.7069 | Regression_pt: 3.2119 | Regression_sin_phi: 1.1968 | LR: 9.95e-04 | DataLoad Time: 0.0820s | Model Forward Time: 1.1696s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0742 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1003 | max_tensorboard_logging: 0.3536 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 3019.4 | uss_MB: 2505.3 | child_rss_MB: 3239.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:21:55,027] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:13,024] INFO : Step 2500/50000 rank0 | Train Loss: 13.4616 | Train Loss Unweighted: 4.7062 | Train Loss Components: Classification: 0.0347 | Classification_binary: 4.1132 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2645 | Regression_eta: 0.0120 | Regression_pt: 0.2632 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3070 | Classification_binary: 4.1132 | Regression_cos_phi: 0.6033 | Regression_energy: 2.8913 | Regression_eta: 0.5356 | Regression_pt: 3.1271 | Regression_sin_phi: 0.8840 | LR: 9.94e-04 | DataLoad Time: 0.1600s | Model Forward Time: 1.1785s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0544 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0340 | max_optimizer_step: 0.1007 | max_tensorboard_logging: 0.3547 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 3020.4 | uss_MB: 2507.2 | child_rss_MB: 3240.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:22:13,235] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:31,263] INFO : Step 2600/50000 rank0 | Train Loss: 12.8478 | Train Loss Unweighted: 4.6747 | Train Loss Components: Classification: 0.0327 | Classification_binary: 4.1557 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2277 | Regression_eta: 0.0131 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2321 | Classification_binary: 4.1557 | Regression_cos_phi: 0.5823 | Regression_energy: 2.4894 | Regression_eta: 0.5857 | Regression_pt: 2.6670 | Regression_sin_phi: 1.1356 | LR: 9.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2777s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0887 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0385 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.3603 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 3020.4 | uss_MB: 2509.5 | child_rss_MB: 3234.5 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:22:31,477] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:49,360] INFO : Step 2700/50000 rank0 | Train Loss: 14.1400 | Train Loss Unweighted: 4.8479 | Train Loss Components: Classification: 0.0331 | Classification_binary: 4.2176 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2833 | Regression_eta: 0.0135 | Regression_pt: 0.2793 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2448 | Classification_binary: 4.2176 | Regression_cos_phi: 0.7333 | Regression_energy: 3.0968 | Regression_eta: 0.6017 | Regression_pt: 3.3188 | Regression_sin_phi: 0.9269 | LR: 9.94e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2529s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0534 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.4032 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 2884.9 | uss_MB: 2373.5 | child_rss_MB: 3241.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:22:49,575] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:07,671] INFO : Step 2800/50000 rank0 | Train Loss: 13.1247 | Train Loss Unweighted: 4.7647 | Train Loss Components: Classification: 0.0322 | Classification_binary: 4.2320 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2347 | Regression_eta: 0.0182 | Regression_pt: 0.2277 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2111 | Classification_binary: 4.2320 | Regression_cos_phi: 0.6339 | Regression_energy: 2.5664 | Regression_eta: 0.8130 | Regression_pt: 2.7051 | Regression_sin_phi: 0.9632 | LR: 9.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1158s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0031 | max_backward: 0.0337 | max_optimizer_step: 0.1016 | max_tensorboard_logging: 0.3043 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 2885.9 | uss_MB: 2374.0 | child_rss_MB: 3239.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:23:07,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:25,584] INFO : Step 2900/50000 rank0 | Train Loss: 12.5833 | Train Loss Unweighted: 4.7099 | Train Loss Components: Classification: 0.0311 | Classification_binary: 4.1574 | Regression_cos_phi: 0.0070 | Regression_energy: 0.2491 | Regression_eta: 0.0104 | Regression_pt: 0.2478 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1708 | Classification_binary: 4.1574 | Regression_cos_phi: 0.4525 | Regression_energy: 2.7235 | Regression_eta: 0.4654 | Regression_pt: 2.9450 | Regression_sin_phi: 0.6687 | LR: 9.93e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2346s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0728 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.0995 | max_tensorboard_logging: 0.3855 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 2886.9 | uss_MB: 2375.2 | child_rss_MB: 3239.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:23:25,799] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:43,434] INFO : Step 3000/50000 rank0 | Train Loss: 13.6555 | Train Loss Unweighted: 4.7133 | Train Loss Components: Classification: 0.0325 | Classification_binary: 4.1296 | Regression_cos_phi: 0.0128 | Regression_energy: 0.2575 | Regression_eta: 0.0139 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2238 | Classification_binary: 4.1296 | Regression_cos_phi: 0.8335 | Regression_energy: 2.8155 | Regression_eta: 0.6212 | Regression_pt: 3.0475 | Regression_sin_phi: 0.9844 | LR: 9.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1645s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0028 | max_backward: 0.1233 | max_optimizer_step: 0.1014 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 2887.9 | uss_MB: 2378.0 | child_rss_MB: 3240.3 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 11:23:43,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:43,665] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:23:43,746] INFO : Running validation on rank0 +rank0 [2026-10-28 11:23:43,746] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:23:43,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:23:44,935] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:23:45,078] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:23:45,209] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:23:45,281] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:23:45,349] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:23:45,417] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:23:45,486] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:23:45,556] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:23:45,655] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:23:45,776] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:23:45,897] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:23:45,967] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:23:46,035] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:23:46,102] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:23:46,172] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:23:46,241] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:23:46,350] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:23:46,357] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:23:46,360] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:23:46,361] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:23:46,364] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:23:46,364] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:23:46,367] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:23:46,370] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:23:46,371] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:23:46,373] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:23:46,374] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:23:46,377] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:23:46,380] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:23:46,380] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:23:46,383] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:23:46,384] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:23:52,338] INFO : VALIDATION | Step=3000/50000 | Train Loss=13.6555 | Valid Loss=13.5005 | Train Loss Unweighted=4.7133 | Valid Loss Unweighted=4.7697 | Stale=0 | ETA=177.3m +rank0 [2026-10-28 11:23:52,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:52,404] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:23:52,407] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:23:52,450] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:53,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:54,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:55,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:55,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:55,804] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:23:55,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:55,862] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:23:55,865] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:55,933] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:57,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:57,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:58,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:59,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:59,113] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:23:59,115] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:59,174] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:23:59,177] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:23:59,249] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:24:00,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:24:01,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:24:01,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:24:02,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:24:02,352] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:24:02,360] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:24:02,613] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:24:03,020] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:04,458] INFO : Plotted jets +rank0 [2026-10-28 11:24:05,543] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:05,575] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:24:05,801] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:24:06,574] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:07,960] INFO : Plotted jets +rank0 [2026-10-28 11:24:09,063] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:09,096] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:24:09,297] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:24:09,658] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:11,031] INFO : Plotted jets +rank0 [2026-10-28 11:24:12,081] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:30,059] INFO : Step 3100/50000 rank0 | Train Loss: 13.7812 | Train Loss Unweighted: 4.7819 | Train Loss Components: Classification: 0.0338 | Classification_binary: 4.2095 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2541 | Regression_eta: 0.0219 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2730 | Classification_binary: 4.2095 | Regression_cos_phi: 0.6530 | Regression_energy: 2.7781 | Regression_eta: 0.9765 | Regression_pt: 2.8723 | Regression_sin_phi: 1.0188 | LR: 9.91e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1537 | max_tensorboard_logging: 0.2888 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 3107.0 | uss_MB: 2595.8 | child_rss_MB: 3237.0 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:24:30,281] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:48,037] INFO : Step 3200/50000 rank0 | Train Loss: 12.8622 | Train Loss Unweighted: 4.6798 | Train Loss Components: Classification: 0.0343 | Classification_binary: 4.1616 | Regression_cos_phi: 0.0103 | Regression_energy: 0.2233 | Regression_eta: 0.0180 | Regression_pt: 0.2233 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2918 | Classification_binary: 4.1616 | Regression_cos_phi: 0.6712 | Regression_energy: 2.4413 | Regression_eta: 0.8037 | Regression_pt: 2.6529 | Regression_sin_phi: 0.8397 | LR: 9.91e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.0962s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0145 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3107.0 | uss_MB: 2596.0 | child_rss_MB: 3237.9 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:24:48,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:05,931] INFO : Step 3300/50000 rank0 | Train Loss: 14.4116 | Train Loss Unweighted: 4.9272 | Train Loss Components: Classification: 0.0332 | Classification_binary: 4.2930 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2841 | Regression_eta: 0.0159 | Regression_pt: 0.2791 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2483 | Classification_binary: 4.2930 | Regression_cos_phi: 0.7250 | Regression_energy: 3.1056 | Regression_eta: 0.7105 | Regression_pt: 3.3168 | Regression_sin_phi: 1.0125 | LR: 9.90e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2383s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0149 | max_loss: 0.0555 | max_backward: 0.0331 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3108.0 | uss_MB: 2595.2 | child_rss_MB: 3264.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:25:06,146] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:23,832] INFO : Step 3400/50000 rank0 | Train Loss: 13.8794 | Train Loss Unweighted: 4.8292 | Train Loss Components: Classification: 0.0305 | Classification_binary: 4.2195 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2771 | Regression_eta: 0.0179 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1474 | Classification_binary: 4.2195 | Regression_cos_phi: 0.6631 | Regression_energy: 3.0294 | Regression_eta: 0.7988 | Regression_pt: 3.1457 | Regression_sin_phi: 0.8754 | LR: 9.90e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1968s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.2937 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3108.0 | uss_MB: 2596.6 | child_rss_MB: 3236.3 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:25:24,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:41,735] INFO : Step 3500/50000 rank0 | Train Loss: 13.1237 | Train Loss Unweighted: 4.5974 | Train Loss Components: Classification: 0.0309 | Classification_binary: 4.0277 | Regression_cos_phi: 0.0107 | Regression_energy: 0.2556 | Regression_eta: 0.0130 | Regression_pt: 0.2501 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1643 | Classification_binary: 4.0277 | Regression_cos_phi: 0.6955 | Regression_energy: 2.7943 | Regression_eta: 0.5778 | Regression_pt: 2.9718 | Regression_sin_phi: 0.8923 | LR: 9.89e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.0591s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0139 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.0983 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3108.0 | uss_MB: 2596.9 | child_rss_MB: 3237.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:25:41,949] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:59,473] INFO : Step 3600/50000 rank0 | Train Loss: 12.9620 | Train Loss Unweighted: 4.6313 | Train Loss Components: Classification: 0.0317 | Classification_binary: 4.1105 | Regression_cos_phi: 0.0097 | Regression_energy: 0.2267 | Regression_eta: 0.0229 | Regression_pt: 0.2201 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1951 | Classification_binary: 4.1105 | Regression_cos_phi: 0.6278 | Regression_energy: 2.4788 | Regression_eta: 1.0226 | Regression_pt: 2.6149 | Regression_sin_phi: 0.9124 | LR: 9.89e-04 | DataLoad Time: 0.1361s | Model Forward Time: 1.1387s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0029 | max_backward: 0.0905 | max_optimizer_step: 0.1073 | max_tensorboard_logging: 0.2735 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3108.0 | uss_MB: 2597.1 | child_rss_MB: 3235.9 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:25:59,685] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:17,266] INFO : Step 3700/50000 rank0 | Train Loss: 14.1425 | Train Loss Unweighted: 4.6245 | Train Loss Components: Classification: 0.0324 | Classification_binary: 4.0383 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2574 | Regression_eta: 0.0311 | Regression_pt: 0.2432 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2193 | Classification_binary: 4.0383 | Regression_cos_phi: 0.6564 | Regression_energy: 2.8140 | Regression_eta: 1.3850 | Regression_pt: 2.8897 | Regression_sin_phi: 1.1397 | LR: 9.88e-04 | DataLoad Time: 0.0759s | Model Forward Time: 1.2308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0542 | max_forward: 0.0167 | max_loss: 0.0031 | max_backward: 0.0343 | max_optimizer_step: 0.1022 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2967.5 | uss_MB: 2455.5 | child_rss_MB: 3232.5 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:26:17,479] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:34,812] INFO : Step 3800/50000 rank0 | Train Loss: 12.3815 | Train Loss Unweighted: 4.5512 | Train Loss Components: Classification: 0.0297 | Classification_binary: 4.0314 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2314 | Regression_eta: 0.0127 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1170 | Classification_binary: 4.0314 | Regression_cos_phi: 0.6206 | Regression_energy: 2.5300 | Regression_eta: 0.5665 | Regression_pt: 2.7079 | Regression_sin_phi: 0.8082 | LR: 9.87e-04 | DataLoad Time: 0.0381s | Model Forward Time: 1.0597s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1024 | max_tensorboard_logging: 0.2692 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 2967.5 | uss_MB: 2456.1 | child_rss_MB: 3229.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:26:35,026] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:52,550] INFO : Step 3900/50000 rank0 | Train Loss: 13.5429 | Train Loss Unweighted: 4.6169 | Train Loss Components: Classification: 0.0320 | Classification_binary: 4.0450 | Regression_cos_phi: 0.0095 | Regression_energy: 0.2535 | Regression_eta: 0.0220 | Regression_pt: 0.2441 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2033 | Classification_binary: 4.0450 | Regression_cos_phi: 0.6140 | Regression_energy: 2.7715 | Regression_eta: 0.9798 | Regression_pt: 2.9000 | Regression_sin_phi: 1.0294 | LR: 9.87e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1339s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0132 | max_loss: 0.0030 | max_backward: 0.1077 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2967.5 | uss_MB: 2456.5 | child_rss_MB: 3232.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:26:52,762] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:10,376] INFO : Step 4000/50000 rank0 | Train Loss: 12.6203 | Train Loss Unweighted: 4.5476 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.9891 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2513 | Regression_eta: 0.0106 | Regression_pt: 0.2507 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1068 | Classification_binary: 3.9891 | Regression_cos_phi: 0.5316 | Regression_energy: 2.7474 | Regression_eta: 0.4707 | Regression_pt: 2.9783 | Regression_sin_phi: 0.7963 | LR: 9.86e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0138 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1014 | max_tensorboard_logging: 0.2795 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 2969.5 | uss_MB: 2456.9 | child_rss_MB: 3231.8 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:10,590] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:10,618] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:27:10,699] INFO : Running validation on rank0 +rank0 [2026-10-28 11:27:10,699] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:27:10,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:11,794] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:27:11,937] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:12,069] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:12,143] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:12,212] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:12,280] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:12,349] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:12,418] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:12,520] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:12,642] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:12,764] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:12,836] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:12,905] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:12,972] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:13,040] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:13,109] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:13,221] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:13,228] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:13,231] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:13,234] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:13,237] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:13,238] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:13,240] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:13,243] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:13,246] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:13,247] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:13,250] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:13,253] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:13,256] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:13,259] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:13,261] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:13,262] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:19,160] INFO : VALIDATION | Step=4000/50000 | Train Loss=12.6203 | Valid Loss=12.6961 | Train Loss Unweighted=4.5476 | Valid Loss Unweighted=4.5506 | Stale=0 | ETA=169.8m +rank0 [2026-10-28 11:27:19,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:19,226] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:19,229] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:19,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:20,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:21,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:21,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:22,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:22,599] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:22,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:22,657] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:22,660] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:22,736] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:24,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:24,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:25,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:25,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:25,876] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:25,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:25,933] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:25,936] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:27:26,009] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:27,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:27,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:28,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:29,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:29,127] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:29,135] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:29,391] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:29,797] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:31,229] INFO : Plotted jets +rank0 [2026-10-28 11:27:32,303] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:32,334] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:32,566] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:33,370] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:34,756] INFO : Plotted jets +rank0 [2026-10-28 11:27:35,849] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:35,880] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:36,085] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:36,444] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:37,814] INFO : Plotted jets +rank0 [2026-10-28 11:27:38,857] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:56,313] INFO : Step 4100/50000 rank0 | Train Loss: 12.7175 | Train Loss Unweighted: 4.4683 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.9477 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2289 | Regression_eta: 0.0237 | Regression_pt: 0.2206 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0760 | Classification_binary: 3.9477 | Regression_cos_phi: 0.6020 | Regression_energy: 2.5024 | Regression_eta: 1.0548 | Regression_pt: 2.6209 | Regression_sin_phi: 0.9136 | LR: 9.85e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.0743s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0030 | max_backward: 0.0327 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3155.9 | uss_MB: 2644.2 | child_rss_MB: 3232.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:56,527] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:13,755] INFO : Step 4200/50000 rank0 | Train Loss: 12.7824 | Train Loss Unweighted: 4.5597 | Train Loss Components: Classification: 0.0313 | Classification_binary: 4.0394 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2326 | Regression_eta: 0.0169 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1772 | Classification_binary: 4.0394 | Regression_cos_phi: 0.7277 | Regression_energy: 2.5426 | Regression_eta: 0.7518 | Regression_pt: 2.5943 | Regression_sin_phi: 0.9494 | LR: 9.84e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.0889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3155.9 | uss_MB: 2644.4 | child_rss_MB: 3231.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:28:13,966] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:31,246] INFO : Step 4300/50000 rank0 | Train Loss: 12.1751 | Train Loss Unweighted: 4.4215 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.9301 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2166 | Regression_eta: 0.0202 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1190 | Classification_binary: 3.9301 | Regression_cos_phi: 0.5484 | Regression_energy: 2.3685 | Regression_eta: 0.8998 | Regression_pt: 2.4653 | Regression_sin_phi: 0.8440 | LR: 9.84e-04 | DataLoad Time: 0.0361s | Model Forward Time: 1.0886s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0145 | max_loss: 0.0037 | max_backward: 0.0331 | max_optimizer_step: 0.1052 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3155.9 | uss_MB: 2644.7 | child_rss_MB: 3226.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:28:31,459] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:48,655] INFO : Step 4400/50000 rank0 | Train Loss: 11.7903 | Train Loss Unweighted: 4.4421 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.9736 | Regression_cos_phi: 0.0100 | Regression_energy: 0.2058 | Regression_eta: 0.0112 | Regression_pt: 0.2042 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0110 | Classification_binary: 3.9736 | Regression_cos_phi: 0.6519 | Regression_energy: 2.2504 | Regression_eta: 0.4998 | Regression_pt: 2.4262 | Regression_sin_phi: 0.9774 | LR: 9.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2815s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0744 | max_forward: 0.0168 | max_loss: 0.0031 | max_backward: 0.0385 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.3834 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3155.9 | uss_MB: 2646.9 | child_rss_MB: 3227.2 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:28:48,869] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:06,118] INFO : Step 4500/50000 rank0 | Train Loss: 13.2269 | Train Loss Unweighted: 4.4148 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.8906 | Regression_cos_phi: 0.0135 | Regression_energy: 0.2314 | Regression_eta: 0.0296 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1301 | Classification_binary: 3.8906 | Regression_cos_phi: 0.8787 | Regression_energy: 2.5300 | Regression_eta: 1.3196 | Regression_pt: 2.4848 | Regression_sin_phi: 0.9931 | LR: 9.82e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2645s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0937 | max_forward: 0.0152 | max_loss: 0.0031 | max_backward: 0.0340 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3156.9 | uss_MB: 2645.4 | child_rss_MB: 3227.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:06,329] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:23,668] INFO : Step 4600/50000 rank0 | Train Loss: 11.8457 | Train Loss Unweighted: 4.4047 | Train Loss Components: Classification: 0.0306 | Classification_binary: 3.9345 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2053 | Regression_eta: 0.0134 | Regression_pt: 0.2018 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1520 | Classification_binary: 3.9345 | Regression_cos_phi: 0.6371 | Regression_energy: 2.2443 | Regression_eta: 0.5995 | Regression_pt: 2.3983 | Regression_sin_phi: 0.8800 | LR: 9.81e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0664s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.0998 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3156.9 | uss_MB: 2645.6 | child_rss_MB: 3230.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:29:23,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:40,732] INFO : Step 4700/50000 rank0 | Train Loss: 12.5723 | Train Loss Unweighted: 4.4591 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.9282 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2397 | Regression_eta: 0.0173 | Regression_pt: 0.2262 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1014 | Classification_binary: 3.9282 | Regression_cos_phi: 0.6256 | Regression_energy: 2.6203 | Regression_eta: 0.7707 | Regression_pt: 2.6877 | Regression_sin_phi: 0.8384 | LR: 9.81e-04 | DataLoad Time: 0.0924s | Model Forward Time: 1.2058s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.1194 | max_optimizer_step: 0.1052 | max_tensorboard_logging: 0.3005 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3046.5 | uss_MB: 2535.4 | child_rss_MB: 3230.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:40,945] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:57,852] INFO : Step 4800/50000 rank0 | Train Loss: 13.5812 | Train Loss Unweighted: 4.6317 | Train Loss Components: Classification: 0.0332 | Classification_binary: 4.0582 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2558 | Regression_eta: 0.0178 | Regression_pt: 0.2446 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2507 | Classification_binary: 4.0582 | Regression_cos_phi: 0.6892 | Regression_energy: 2.7963 | Regression_eta: 0.7948 | Regression_pt: 2.9064 | Regression_sin_phi: 1.0856 | LR: 9.80e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.2148s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0307 | max_forward: 0.0151 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1034 | max_tensorboard_logging: 0.3824 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3048.5 | uss_MB: 2537.8 | child_rss_MB: 3227.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:58,066] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:15,587] INFO : Step 4900/50000 rank0 | Train Loss: 12.2599 | Train Loss Unweighted: 4.4027 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.8965 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2228 | Regression_eta: 0.0148 | Regression_pt: 0.2188 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1582 | Classification_binary: 3.8965 | Regression_cos_phi: 0.6606 | Regression_energy: 2.4357 | Regression_eta: 0.6595 | Regression_pt: 2.5992 | Regression_sin_phi: 0.8502 | LR: 9.79e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0145 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3048.5 | uss_MB: 2538.0 | child_rss_MB: 3223.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:30:15,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:32,832] INFO : Step 5000/50000 rank0 | Train Loss: 11.6048 | Train Loss Unweighted: 4.3258 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.8422 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2146 | Regression_eta: 0.0118 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1125 | Classification_binary: 3.8422 | Regression_cos_phi: 0.5225 | Regression_energy: 2.3466 | Regression_eta: 0.5254 | Regression_pt: 2.5163 | Regression_sin_phi: 0.7392 | LR: 9.78e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.0821s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1009 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3048.5 | uss_MB: 2538.1 | child_rss_MB: 3226.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:30:33,044] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:33,081] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:30:33,165] INFO : Running validation on rank0 +rank0 [2026-10-28 11:30:33,165] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:30:33,166] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:30:34,255] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:30:34,399] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:30:34,529] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:30:34,602] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:30:34,671] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:30:34,738] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:30:34,806] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:30:34,875] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:30:34,972] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:30:35,102] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:30:35,230] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:30:35,301] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:30:35,370] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:30:35,438] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:30:35,506] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:30:35,575] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:30:35,686] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:30:35,693] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:30:35,696] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:30:35,698] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:30:35,699] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:30:35,702] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:30:35,705] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:30:35,705] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:30:35,708] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:30:35,711] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:30:35,714] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:30:35,716] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:30:35,719] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:30:35,720] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:30:35,723] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:30:35,726] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:30:41,638] INFO : VALIDATION | Step=5000/50000 | Train Loss=11.6048 | Valid Loss=12.2601 | Train Loss Unweighted=4.3258 | Valid Loss Unweighted=4.4266 | Stale=0 | ETA=163.2m +rank0 [2026-10-28 11:30:41,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:41,705] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:30:41,709] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:30:41,856] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:43,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:44,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:44,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:45,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:45,467] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:30:45,470] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:45,529] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:30:45,532] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:45,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:47,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:47,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:48,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:48,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:48,840] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:48,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:48,898] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:30:48,901] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:30:48,978] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:50,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:50,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:51,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:51,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:51,983] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:52,002] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:30:52,258] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:30:52,666] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:54,109] INFO : Plotted jets +rank0 [2026-10-28 11:30:55,191] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:55,273] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:55,500] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:55,889] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:57,668] INFO : Plotted jets +rank0 [2026-10-28 11:30:58,765] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:58,798] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:30:59,002] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:30:59,365] INFO : Plotted number of elements +rank0 [2026-10-28 11:31:00,738] INFO : Plotted jets +rank0 [2026-10-28 11:31:01,787] INFO : Plotted jet ratio +rank0 [2026-10-28 11:31:19,406] INFO : Step 5100/50000 rank0 | Train Loss: 12.5418 | Train Loss Unweighted: 4.5385 | Train Loss Components: Classification: 0.0303 | Classification_binary: 4.0181 | Regression_cos_phi: 0.0073 | Regression_energy: 0.2299 | Regression_eta: 0.0230 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1406 | Classification_binary: 4.0181 | Regression_cos_phi: 0.4737 | Regression_energy: 2.5133 | Regression_eta: 1.0257 | Regression_pt: 2.6409 | Regression_sin_phi: 0.7296 | LR: 9.77e-04 | DataLoad Time: 0.0400s | Model Forward Time: 1.3004s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1080 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.3934 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3119.7 | uss_MB: 2611.2 | child_rss_MB: 3226.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:31:19,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:36,763] INFO : Step 5200/50000 rank0 | Train Loss: 12.4054 | Train Loss Unweighted: 4.5089 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.9835 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2364 | Regression_eta: 0.0145 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0491 | Classification_binary: 3.9835 | Regression_cos_phi: 0.5765 | Regression_energy: 2.5840 | Regression_eta: 0.6449 | Regression_pt: 2.7191 | Regression_sin_phi: 0.8484 | LR: 9.76e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2828s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0156 | max_loss: 0.1017 | max_backward: 0.0377 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.3877 | max_tensorboard_flush: 0.0038 | Memory: rss_MB: 3120.7 | uss_MB: 2613.3 | child_rss_MB: 3252.8 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:31:36,974] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:53,664] INFO : Step 5300/50000 rank0 | Train Loss: 11.5300 | Train Loss Unweighted: 4.2441 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.7826 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2038 | Regression_eta: 0.0150 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1241 | Classification_binary: 3.7826 | Regression_cos_phi: 0.6065 | Regression_energy: 2.2277 | Regression_eta: 0.6671 | Regression_pt: 2.3182 | Regression_sin_phi: 0.8039 | LR: 9.75e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2122s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0352 | max_forward: 0.0135 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1050 | max_tensorboard_logging: 0.3763 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3121.7 | uss_MB: 2612.9 | child_rss_MB: 3225.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:31:53,878] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:10,789] INFO : Step 5400/50000 rank0 | Train Loss: 12.0754 | Train Loss Unweighted: 4.1915 | Train Loss Components: Classification: 0.0285 | Classification_binary: 3.6673 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2335 | Regression_eta: 0.0150 | Regression_pt: 0.2299 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0710 | Classification_binary: 3.6673 | Regression_cos_phi: 0.5751 | Regression_energy: 2.5529 | Regression_eta: 0.6697 | Regression_pt: 2.7313 | Regression_sin_phi: 0.8081 | LR: 9.74e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.1601s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0140 | max_loss: 0.0029 | max_backward: 0.1367 | max_optimizer_step: 0.1043 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3123.7 | uss_MB: 2613.9 | child_rss_MB: 3227.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:32:11,002] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:27,848] INFO : Step 5500/50000 rank0 | Train Loss: 12.1733 | Train Loss Unweighted: 4.2507 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.7408 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2336 | Regression_eta: 0.0192 | Regression_pt: 0.2094 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1097 | Classification_binary: 3.7408 | Regression_cos_phi: 0.6591 | Regression_energy: 2.5539 | Regression_eta: 0.8562 | Regression_pt: 2.4876 | Regression_sin_phi: 0.7660 | LR: 9.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2267s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0579 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1029 | max_tensorboard_logging: 0.3867 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3124.7 | uss_MB: 2614.4 | child_rss_MB: 3222.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:32:28,062] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:44,706] INFO : Step 5600/50000 rank0 | Train Loss: 11.4903 | Train Loss Unweighted: 4.2869 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.8150 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2073 | Regression_eta: 0.0122 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0270 | Classification_binary: 3.8150 | Regression_cos_phi: 0.5647 | Regression_energy: 2.2669 | Regression_eta: 0.5431 | Regression_pt: 2.4704 | Regression_sin_phi: 0.8033 | LR: 9.72e-04 | DataLoad Time: 0.0628s | Model Forward Time: 1.1990s | Phase Times: max_pre_step_barrier: 0.0953 | max_batch_to_device: 0.0011 | max_forward: 0.0137 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1006 | max_tensorboard_logging: 0.3178 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3124.7 | uss_MB: 2615.0 | child_rss_MB: 3223.1 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:32:44,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:01,827] INFO : Step 5700/50000 rank0 | Train Loss: 12.0141 | Train Loss Unweighted: 4.3467 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.8520 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2208 | Regression_eta: 0.0154 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0490 | Classification_binary: 3.8520 | Regression_cos_phi: 0.6221 | Regression_energy: 2.4142 | Regression_eta: 0.6869 | Regression_pt: 2.5164 | Regression_sin_phi: 0.8734 | LR: 9.71e-04 | DataLoad Time: 0.0564s | Model Forward Time: 1.2105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0153 | max_loss: 0.0030 | max_backward: 0.1352 | max_optimizer_step: 0.1053 | max_tensorboard_logging: 0.3005 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3122.1 | uss_MB: 2612.4 | child_rss_MB: 3221.3 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:33:02,042] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:18,856] INFO : Step 5800/50000 rank0 | Train Loss: 12.0533 | Train Loss Unweighted: 4.3016 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.8058 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2201 | Regression_eta: 0.0159 | Regression_pt: 0.2123 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0453 | Classification_binary: 3.8058 | Regression_cos_phi: 0.6593 | Regression_energy: 2.4057 | Regression_eta: 0.7111 | Regression_pt: 2.5225 | Regression_sin_phi: 0.9036 | LR: 9.70e-04 | DataLoad Time: 0.0100s | Model Forward Time: 1.2515s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0576 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1025 | max_tensorboard_logging: 0.4215 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3122.1 | uss_MB: 2613.6 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:33:19,069] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:35,648] INFO : Step 5900/50000 rank0 | Train Loss: 11.9432 | Train Loss Unweighted: 4.2404 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.7576 | Regression_cos_phi: 0.0118 | Regression_energy: 0.2127 | Regression_eta: 0.0138 | Regression_pt: 0.2049 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1289 | Classification_binary: 3.7576 | Regression_cos_phi: 0.7637 | Regression_energy: 2.3249 | Regression_eta: 0.6153 | Regression_pt: 2.4343 | Regression_sin_phi: 0.9185 | LR: 9.69e-04 | DataLoad Time: 0.0441s | Model Forward Time: 1.2331s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0722 | max_forward: 0.0176 | max_loss: 0.0031 | max_backward: 0.0389 | max_optimizer_step: 0.1003 | max_tensorboard_logging: 0.3693 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3123.1 | uss_MB: 2615.9 | child_rss_MB: 3217.5 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:33:35,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:52,771] INFO : Step 6000/50000 rank0 | Train Loss: 11.6470 | Train Loss Unweighted: 4.3060 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.8349 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2074 | Regression_eta: 0.0118 | Regression_pt: 0.2035 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1198 | Classification_binary: 3.8349 | Regression_cos_phi: 0.6014 | Regression_energy: 2.2679 | Regression_eta: 0.5273 | Regression_pt: 2.4183 | Regression_sin_phi: 0.8775 | LR: 9.68e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.1880s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0032 | max_backward: 0.0384 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3125.1 | uss_MB: 2617.3 | child_rss_MB: 3218.6 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:33:52,998] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:53,016] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:33:53,097] INFO : Running validation on rank0 +rank0 [2026-10-28 11:33:53,097] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:33:53,097] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:33:54,298] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:33:54,441] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:33:54,572] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:33:54,645] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:33:54,713] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:33:54,780] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:33:54,851] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:33:54,919] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:33:55,022] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:33:55,151] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:33:55,279] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:33:55,349] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:33:55,417] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:33:55,484] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:33:55,553] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:33:55,622] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:33:55,736] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:33:55,743] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:33:55,746] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:33:55,747] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:33:55,749] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:33:55,750] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:33:55,753] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:33:55,756] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:33:55,757] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:33:55,759] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:33:55,762] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:33:55,763] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:33:55,765] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:33:55,766] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:33:55,769] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:33:55,771] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:34:01,695] INFO : VALIDATION | Step=6000/50000 | Train Loss=11.6470 | Valid Loss=11.7646 | Train Loss Unweighted=4.3060 | Valid Loss Unweighted=4.2717 | Stale=0 | ETA=157.5m +rank0 [2026-10-28 11:34:01,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:01,763] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:34:01,766] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:01,848] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:03,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:03,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:04,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:05,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:05,244] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:05,247] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:05,302] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:34:05,314] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:05,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:06,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:07,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:07,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:08,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:08,565] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:34:08,568] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:08,621] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:34:08,624] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:34:08,701] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:10,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:10,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:11,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:11,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:11,700] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:34:11,723] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:11,983] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:12,388] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:13,825] INFO : Plotted jets +rank0 [2026-10-28 11:34:14,910] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:14,942] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:15,171] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:15,560] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:17,392] INFO : Plotted jets +rank0 [2026-10-28 11:34:18,496] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:18,531] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:34:18,737] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:34:19,099] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:20,474] INFO : Plotted jets +rank0 [2026-10-28 11:34:21,523] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:38,547] INFO : Step 6100/50000 rank0 | Train Loss: 11.5079 | Train Loss Unweighted: 4.2354 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.7590 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2107 | Regression_eta: 0.0130 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0237 | Classification_binary: 3.7590 | Regression_cos_phi: 0.5348 | Regression_energy: 2.3033 | Regression_eta: 0.5796 | Regression_pt: 2.4779 | Regression_sin_phi: 0.8296 | LR: 9.67e-04 | DataLoad Time: 0.0507s | Model Forward Time: 1.1045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.2942 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3194.6 | uss_MB: 2685.1 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:34:38,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:55,337] INFO : Step 6200/50000 rank0 | Train Loss: 12.1244 | Train Loss Unweighted: 4.2133 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.7298 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2126 | Regression_eta: 0.0170 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0543 | Classification_binary: 3.7298 | Regression_cos_phi: 0.7927 | Regression_energy: 2.3248 | Regression_eta: 0.7587 | Regression_pt: 2.4053 | Regression_sin_phi: 1.0588 | LR: 9.66e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1052s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3194.6 | uss_MB: 2686.5 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:55,550] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:12,180] INFO : Step 6300/50000 rank0 | Train Loss: 12.0051 | Train Loss Unweighted: 4.1784 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.6932 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2134 | Regression_eta: 0.0237 | Regression_pt: 0.2034 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9352 | Classification_binary: 3.6932 | Regression_cos_phi: 0.6728 | Regression_energy: 2.3334 | Regression_eta: 1.0574 | Regression_pt: 2.4167 | Regression_sin_phi: 0.8965 | LR: 9.65e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1401s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.3232 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3194.6 | uss_MB: 2686.9 | child_rss_MB: 3212.3 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:35:12,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:29,116] INFO : Step 6400/50000 rank0 | Train Loss: 12.0367 | Train Loss Unweighted: 4.2760 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.7806 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2195 | Regression_eta: 0.0153 | Regression_pt: 0.2133 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0028 | Classification_binary: 3.7806 | Regression_cos_phi: 0.6801 | Regression_energy: 2.4001 | Regression_eta: 0.6841 | Regression_pt: 2.5341 | Regression_sin_phi: 0.9548 | LR: 9.64e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.1421s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0033 | max_backward: 0.1012 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3194.6 | uss_MB: 2687.2 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:35:29,329] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:45,606] INFO : Step 6500/50000 rank0 | Train Loss: 11.4772 | Train Loss Unweighted: 4.1222 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.6574 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2052 | Regression_eta: 0.0184 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0450 | Classification_binary: 3.6574 | Regression_cos_phi: 0.5759 | Regression_energy: 2.2436 | Regression_eta: 0.8191 | Regression_pt: 2.3294 | Regression_sin_phi: 0.8068 | LR: 9.63e-04 | DataLoad Time: 0.0209s | Model Forward Time: 1.1951s | Phase Times: max_pre_step_barrier: 0.0952 | max_batch_to_device: 0.0011 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2929 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3194.6 | uss_MB: 2687.3 | child_rss_MB: 3214.1 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:35:45,820] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:02,397] INFO : Step 6600/50000 rank0 | Train Loss: 11.7705 | Train Loss Unweighted: 4.1216 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.6391 | Regression_cos_phi: 0.0091 | Regression_energy: 0.2146 | Regression_eta: 0.0192 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9663 | Classification_binary: 3.6391 | Regression_cos_phi: 0.5903 | Regression_energy: 2.3463 | Regression_eta: 0.8574 | Regression_pt: 2.4224 | Regression_sin_phi: 0.9486 | LR: 9.62e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2680s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0901 | max_forward: 0.0139 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1049 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 3194.6 | uss_MB: 2687.6 | child_rss_MB: 3214.0 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:36:02,610] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:19,207] INFO : Step 6700/50000 rank0 | Train Loss: 10.8334 | Train Loss Unweighted: 4.0381 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.5865 | Regression_cos_phi: 0.0080 | Regression_energy: 0.2014 | Regression_eta: 0.0117 | Regression_pt: 0.1976 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9584 | Classification_binary: 3.5865 | Regression_cos_phi: 0.5190 | Regression_energy: 2.2018 | Regression_eta: 0.5203 | Regression_pt: 2.3483 | Regression_sin_phi: 0.6990 | LR: 9.61e-04 | DataLoad Time: 0.1538s | Model Forward Time: 1.0918s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1007 | max_tensorboard_logging: 0.2810 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3194.6 | uss_MB: 2687.7 | child_rss_MB: 3212.4 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:36:19,421] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:35,592] INFO : Step 6800/50000 rank0 | Train Loss: 11.1651 | Train Loss Unweighted: 4.0179 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.5472 | Regression_cos_phi: 0.0078 | Regression_energy: 0.2105 | Regression_eta: 0.0112 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9353 | Classification_binary: 3.5472 | Regression_cos_phi: 0.5057 | Regression_energy: 2.3011 | Regression_eta: 0.5013 | Regression_pt: 2.4541 | Regression_sin_phi: 0.9204 | LR: 9.60e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1450s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0866 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3195.8 | uss_MB: 2686.8 | child_rss_MB: 3215.5 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:36:35,808] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:52,040] INFO : Step 6900/50000 rank0 | Train Loss: 11.5291 | Train Loss Unweighted: 4.1205 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.6531 | Regression_cos_phi: 0.0115 | Regression_energy: 0.2070 | Regression_eta: 0.0157 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0188 | Classification_binary: 3.6531 | Regression_cos_phi: 0.7435 | Regression_energy: 2.2632 | Regression_eta: 0.7006 | Regression_pt: 2.3493 | Regression_sin_phi: 0.8006 | LR: 9.58e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2153s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0354 | max_forward: 0.0165 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.3678 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3196.8 | uss_MB: 2690.2 | child_rss_MB: 3214.6 | allocated_MB: 210.7 | reserved_MB: 2738.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:36:52,252] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:08,724] INFO : Step 7000/50000 rank0 | Train Loss: 12.5052 | Train Loss Unweighted: 4.2046 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.6562 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2468 | Regression_eta: 0.0138 | Regression_pt: 0.2406 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9552 | Classification_binary: 3.6562 | Regression_cos_phi: 0.7365 | Regression_energy: 2.6980 | Regression_eta: 0.6133 | Regression_pt: 2.8592 | Regression_sin_phi: 0.9868 | LR: 9.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2394s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0602 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1053 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3196.8 | uss_MB: 2688.2 | child_rss_MB: 3210.5 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:37:08,938] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:08,974] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 11:37:09,154] INFO : Running validation on rank0 +rank0 [2026-10-28 11:37:09,155] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:37:09,155] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:37:10,274] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 11:37:10,418] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:37:10,550] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:37:10,622] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:37:10,691] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:37:10,758] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:37:10,827] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:37:10,895] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:37:10,997] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:37:11,118] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:37:11,240] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:37:11,312] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:37:11,380] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:37:11,448] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:37:11,516] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:37:11,585] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:37:11,703] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:37:11,710] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:37:11,713] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:37:11,716] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:37:11,717] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:37:11,719] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:37:11,720] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:37:11,723] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:37:11,724] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:37:11,727] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:37:11,729] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:37:11,730] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:37:11,733] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:37:11,735] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:37:11,738] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:37:11,739] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:37:17,766] INFO : VALIDATION | Step=7000/50000 | Train Loss=12.5052 | Valid Loss=11.4875 | Train Loss Unweighted=4.2046 | Valid Loss Unweighted=4.1305 | Stale=0 | ETA=152.0m +rank0 [2026-10-28 11:37:17,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:17,832] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:37:17,836] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:37:17,980] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:19,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:20,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:20,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:21,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:21,492] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:37:21,495] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:21,549] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:37:21,552] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:21,633] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:23,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:23,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:24,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:24,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:24,861] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:24,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:24,916] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:37:24,919] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:37:25,002] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:26,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:27,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:27,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:28,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:28,170] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:28,177] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:37:28,437] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:37:28,842] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:30,284] INFO : Plotted jets +rank0 [2026-10-28 11:37:31,363] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:31,395] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:31,627] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:32,018] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:33,848] INFO : Plotted jets +rank0 [2026-10-28 11:37:34,958] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:34,990] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 11:37:35,196] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 11:37:35,557] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:36,929] INFO : Plotted jets +rank0 [2026-10-28 11:37:37,979] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:54,650] INFO : Step 7100/50000 rank0 | Train Loss: 11.8157 | Train Loss Unweighted: 4.1552 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.6620 | Regression_cos_phi: 0.0091 | Regression_energy: 0.2219 | Regression_eta: 0.0194 | Regression_pt: 0.2101 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8794 | Classification_binary: 3.6620 | Regression_cos_phi: 0.5901 | Regression_energy: 2.4255 | Regression_eta: 0.8638 | Regression_pt: 2.4961 | Regression_sin_phi: 0.8988 | LR: 9.56e-04 | DataLoad Time: 0.1463s | Model Forward Time: 1.1802s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0695 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3409.6 | uss_MB: 2903.9 | child_rss_MB: 3219.1 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:37:54,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:11,025] INFO : Step 7200/50000 rank0 | Train Loss: 10.5667 | Train Loss Unweighted: 4.1051 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.6800 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1898 | Regression_eta: 0.0120 | Regression_pt: 0.1846 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8921 | Classification_binary: 3.6800 | Regression_cos_phi: 0.5112 | Regression_energy: 2.0752 | Regression_eta: 0.5333 | Regression_pt: 2.1939 | Regression_sin_phi: 0.6811 | LR: 9.55e-04 | DataLoad Time: 0.1264s | Model Forward Time: 1.3279s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1046 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1097 | max_tensorboard_logging: 0.3890 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3268.3 | uss_MB: 2763.5 | child_rss_MB: 3216.8 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:38:11,238] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:27,752] INFO : Step 7300/50000 rank0 | Train Loss: 11.2619 | Train Loss Unweighted: 4.1059 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.6374 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2073 | Regression_eta: 0.0130 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9568 | Classification_binary: 3.6374 | Regression_cos_phi: 0.5632 | Regression_energy: 2.2663 | Regression_eta: 0.5807 | Regression_pt: 2.4414 | Regression_sin_phi: 0.8162 | LR: 9.53e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2615s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0685 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1074 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3268.3 | uss_MB: 2764.1 | child_rss_MB: 3211.0 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:38:27,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:44,225] INFO : Step 7400/50000 rank0 | Train Loss: 11.8457 | Train Loss Unweighted: 4.0443 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.5303 | Regression_cos_phi: 0.0088 | Regression_energy: 0.2313 | Regression_eta: 0.0198 | Regression_pt: 0.2211 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9391 | Classification_binary: 3.5303 | Regression_cos_phi: 0.5685 | Regression_energy: 2.5286 | Regression_eta: 0.8819 | Regression_pt: 2.6269 | Regression_sin_phi: 0.7703 | LR: 9.52e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2088s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0030 | max_backward: 0.1410 | max_optimizer_step: 0.1088 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3268.3 | uss_MB: 2764.2 | child_rss_MB: 3212.6 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:38:44,438] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:00,502] INFO : Step 7500/50000 rank0 | Train Loss: 11.9213 | Train Loss Unweighted: 4.1695 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.6741 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2198 | Regression_eta: 0.0159 | Regression_pt: 0.2119 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0455 | Classification_binary: 3.6741 | Regression_cos_phi: 0.6857 | Regression_energy: 2.4035 | Regression_eta: 0.7078 | Regression_pt: 2.5182 | Regression_sin_phi: 0.8865 | LR: 9.51e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2418s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0134 | max_loss: 0.0577 | max_backward: 0.0331 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3268.3 | uss_MB: 2763.3 | child_rss_MB: 3212.4 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:39:00,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:16,758] INFO : Step 7600/50000 rank0 | Train Loss: 12.9397 | Train Loss Unweighted: 4.2959 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.7543 | Regression_cos_phi: 0.0111 | Regression_energy: 0.2410 | Regression_eta: 0.0237 | Regression_pt: 0.2285 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9725 | Classification_binary: 3.7543 | Regression_cos_phi: 0.7189 | Regression_energy: 2.6352 | Regression_eta: 1.0549 | Regression_pt: 2.7145 | Regression_sin_phi: 1.0894 | LR: 9.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1026s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1021 | max_tensorboard_logging: 0.3074 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3268.3 | uss_MB: 2764.9 | child_rss_MB: 3210.2 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:39:16,969] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:33,154] INFO : Step 7700/50000 rank0 | Train Loss: 11.1650 | Train Loss Unweighted: 4.0008 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.5332 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2076 | Regression_eta: 0.0113 | Regression_pt: 0.2046 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9720 | Classification_binary: 3.5332 | Regression_cos_phi: 0.6023 | Regression_energy: 2.2698 | Regression_eta: 0.5059 | Regression_pt: 2.4310 | Regression_sin_phi: 0.8507 | LR: 9.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.1308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0504 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3268.3 | uss_MB: 2765.0 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:39:33,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:49,446] INFO : Step 7800/50000 rank0 | Train Loss: 11.3556 | Train Loss Unweighted: 4.0260 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2115 | Regression_eta: 0.0140 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9623 | Classification_binary: 3.5479 | Regression_cos_phi: 0.5779 | Regression_energy: 2.3128 | Regression_eta: 0.6251 | Regression_pt: 2.4845 | Regression_sin_phi: 0.8450 | LR: 9.47e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2360s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0307 | max_backward: 0.0347 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.3873 | max_tensorboard_flush: 0.0013 | Memory: rss_MB: 3266.2 | uss_MB: 2763.3 | child_rss_MB: 3211.5 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:39:49,657] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:05,702] INFO : Step 7900/50000 rank0 | Train Loss: 11.4037 | Train Loss Unweighted: 4.0769 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.6122 | Regression_cos_phi: 0.0083 | Regression_energy: 0.2095 | Regression_eta: 0.0174 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9308 | Classification_binary: 3.6122 | Regression_cos_phi: 0.5407 | Regression_energy: 2.2901 | Regression_eta: 0.7768 | Regression_pt: 2.3154 | Regression_sin_phi: 0.9376 | LR: 9.46e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0318 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1068 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3266.2 | uss_MB: 2765.0 | child_rss_MB: 3211.4 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:40:05,914] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:21,826] INFO : Step 8000/50000 rank0 | Train Loss: 10.3619 | Train Loss Unweighted: 3.8947 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.4879 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1770 | Regression_eta: 0.0121 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9382 | Classification_binary: 3.4879 | Regression_cos_phi: 0.6574 | Regression_energy: 1.9353 | Regression_eta: 0.5396 | Regression_pt: 2.0783 | Regression_sin_phi: 0.7253 | LR: 9.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.0875s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 0.2800 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3266.2 | uss_MB: 2765.1 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:40:22,038] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:22,070] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 11:40:22,153] INFO : Running validation on rank0 +rank0 [2026-10-28 11:40:22,153] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:40:22,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:40:23,252] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 11:40:23,397] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:40:23,529] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:40:23,603] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:40:23,672] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:40:23,741] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:40:23,809] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:40:23,879] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:40:23,983] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:40:24,114] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:40:24,244] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:40:24,315] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:40:24,384] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:40:24,451] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:40:24,521] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:40:24,589] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:40:24,690] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:40:24,696] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:40:24,699] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:40:24,702] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:40:24,702] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:40:24,705] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:40:24,706] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:40:24,709] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:40:24,711] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:40:24,712] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:40:24,715] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:40:24,717] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:40:24,720] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:40:24,721] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:40:24,723] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:40:24,724] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:40:30,523] INFO : VALIDATION | Step=8000/50000 | Train Loss=10.3619 | Valid Loss=10.9081 | Train Loss Unweighted=3.8947 | Valid Loss Unweighted=3.9603 | Stale=0 | ETA=146.8m +rank0 [2026-10-28 11:40:30,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:30,590] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:40:30,593] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:40:30,737] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:32,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:32,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:33,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:33,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:33,966] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:33,969] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:34,024] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:40:34,027] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:34,109] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:35,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:36,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:36,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:37,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:37,380] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:37,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:37,437] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:40:37,440] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:40:37,497] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:38,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:39,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:39,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:40,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:40,525] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:40,532] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:40:40,791] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:40:41,195] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:42,637] INFO : Plotted jets +rank0 [2026-10-28 11:40:43,720] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:43,752] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:43,984] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:44,373] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:46,203] INFO : Plotted jets +rank0 [2026-10-28 11:40:47,316] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:47,348] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 11:40:47,551] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 11:40:47,911] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:49,279] INFO : Plotted jets +rank0 [2026-10-28 11:40:50,324] INFO : Plotted jet ratio +rank0 [2026-10-28 11:41:06,629] INFO : Step 8100/50000 rank0 | Train Loss: 10.9651 | Train Loss Unweighted: 3.8570 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.4188 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1917 | Regression_eta: 0.0159 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9515 | Classification_binary: 3.4188 | Regression_cos_phi: 0.5991 | Regression_energy: 2.0960 | Regression_eta: 0.7099 | Regression_pt: 2.2069 | Regression_sin_phi: 0.9830 | LR: 9.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2290s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0185 | max_forward: 0.0173 | max_loss: 0.0034 | max_backward: 0.0383 | max_optimizer_step: 0.1096 | max_tensorboard_logging: 0.3558 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3475.2 | uss_MB: 2977.7 | child_rss_MB: 3211.3 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:41:06,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:22,797] INFO : Step 8200/50000 rank0 | Train Loss: 10.7617 | Train Loss Unweighted: 3.9594 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.5239 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1925 | Regression_eta: 0.0171 | Regression_pt: 0.1874 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8300 | Classification_binary: 3.5239 | Regression_cos_phi: 0.5314 | Regression_energy: 2.1046 | Regression_eta: 0.7645 | Regression_pt: 2.2269 | Regression_sin_phi: 0.7804 | LR: 9.42e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0892s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1073 | max_tensorboard_logging: 0.2773 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3338.3 | uss_MB: 2838.0 | child_rss_MB: 3211.6 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:41:23,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:38,709] INFO : Step 8300/50000 rank0 | Train Loss: 10.9664 | Train Loss Unweighted: 3.9141 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.4512 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2079 | Regression_eta: 0.0116 | Regression_pt: 0.2020 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8419 | Classification_binary: 3.4512 | Regression_cos_phi: 0.7016 | Regression_energy: 2.2731 | Regression_eta: 0.5164 | Regression_pt: 2.4004 | Regression_sin_phi: 0.7819 | LR: 9.40e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2518s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0346 | max_backward: 0.0340 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0132 | Memory: rss_MB: 3339.3 | uss_MB: 2837.2 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:38,921] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:54,992] INFO : Step 8400/50000 rank0 | Train Loss: 10.8453 | Train Loss Unweighted: 4.0634 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.6276 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1939 | Regression_eta: 0.0134 | Regression_pt: 0.1876 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8877 | Classification_binary: 3.6276 | Regression_cos_phi: 0.5672 | Regression_energy: 2.1194 | Regression_eta: 0.5958 | Regression_pt: 2.2293 | Regression_sin_phi: 0.8185 | LR: 9.39e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0691 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3339.3 | uss_MB: 2838.6 | child_rss_MB: 3210.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:55,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:11,161] INFO : Step 8500/50000 rank0 | Train Loss: 11.0323 | Train Loss Unweighted: 3.9909 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.5437 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1970 | Regression_eta: 0.0126 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9783 | Classification_binary: 3.5437 | Regression_cos_phi: 0.5950 | Regression_energy: 2.1538 | Regression_eta: 0.5618 | Regression_pt: 2.2916 | Regression_sin_phi: 0.9081 | LR: 9.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.1425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0031 | max_backward: 0.0339 | max_optimizer_step: 0.1076 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3339.3 | uss_MB: 2838.7 | child_rss_MB: 3208.6 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:11,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:27,170] INFO : Step 8600/50000 rank0 | Train Loss: 10.6909 | Train Loss Unweighted: 3.9331 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.5060 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1862 | Regression_eta: 0.0132 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9486 | Classification_binary: 3.5060 | Regression_cos_phi: 0.5622 | Regression_energy: 2.0354 | Regression_eta: 0.5887 | Regression_pt: 2.1963 | Regression_sin_phi: 0.8538 | LR: 9.36e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1220s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3339.3 | uss_MB: 2838.9 | child_rss_MB: 3209.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:27,383] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:43,248] INFO : Step 8700/50000 rank0 | Train Loss: 11.3930 | Train Loss Unweighted: 3.9739 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.5127 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2046 | Regression_eta: 0.0185 | Regression_pt: 0.1931 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9336 | Classification_binary: 3.5127 | Regression_cos_phi: 0.7125 | Regression_energy: 2.2365 | Regression_eta: 0.8259 | Regression_pt: 2.2944 | Regression_sin_phi: 0.8774 | LR: 9.34e-04 | DataLoad Time: 0.1214s | Model Forward Time: 1.3086s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0552 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0156 | Memory: rss_MB: 3340.3 | uss_MB: 2839.0 | child_rss_MB: 3212.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:42:43,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:59,305] INFO : Step 8800/50000 rank0 | Train Loss: 10.3360 | Train Loss Unweighted: 3.9224 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.5130 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1800 | Regression_eta: 0.0117 | Regression_pt: 0.1756 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9857 | Classification_binary: 3.5130 | Regression_cos_phi: 0.5454 | Regression_energy: 1.9674 | Regression_eta: 0.5230 | Regression_pt: 2.0860 | Regression_sin_phi: 0.7154 | LR: 9.33e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1281s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1047 | max_tensorboard_logging: 0.3010 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3337.7 | uss_MB: 2837.6 | child_rss_MB: 3209.9 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:42:59,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:15,340] INFO : Step 8900/50000 rank0 | Train Loss: 10.7503 | Train Loss Unweighted: 3.9338 | Train Loss Components: Classification: 0.0233 | Classification_binary: 3.4975 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1933 | Regression_eta: 0.0122 | Regression_pt: 0.1889 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8784 | Classification_binary: 3.4975 | Regression_cos_phi: 0.6207 | Regression_energy: 2.1136 | Regression_eta: 0.5439 | Regression_pt: 2.2447 | Regression_sin_phi: 0.8516 | LR: 9.31e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1229s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.0327 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3337.7 | uss_MB: 2839.1 | child_rss_MB: 3212.2 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:43:15,551] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:30,995] INFO : Step 9000/50000 rank0 | Train Loss: 9.9140 | Train Loss Unweighted: 3.7386 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.3382 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1766 | Regression_eta: 0.0097 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9374 | Classification_binary: 3.3382 | Regression_cos_phi: 0.5213 | Regression_energy: 1.9306 | Regression_eta: 0.4330 | Regression_pt: 2.0659 | Regression_sin_phi: 0.6875 | LR: 9.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1891s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0155 | max_loss: 0.0033 | max_backward: 0.1403 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.2737 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3337.7 | uss_MB: 2839.2 | child_rss_MB: 3212.0 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:43:31,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:31,228] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 11:43:31,310] INFO : Running validation on rank0 +rank0 [2026-10-28 11:43:31,310] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:43:31,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:43:32,513] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 11:43:32,656] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:43:32,787] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:43:32,860] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:43:32,928] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:43:32,995] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:43:33,064] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:43:33,133] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:43:33,235] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:43:33,366] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:43:33,495] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:43:33,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:43:33,633] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:43:33,701] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:43:33,771] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:43:33,840] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:43:33,947] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:43:33,954] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:43:33,956] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:43:33,957] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:43:33,960] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:43:33,963] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:43:33,966] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:43:33,967] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:43:33,969] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:43:33,970] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:43:33,973] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:43:33,975] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:43:33,977] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:43:33,979] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:43:33,980] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:43:33,983] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:43:39,919] INFO : VALIDATION | Step=9000/50000 | Train Loss=9.9140 | Valid Loss=10.8381 | Train Loss Unweighted=3.7386 | Valid Loss Unweighted=3.9186 | Stale=0 | ETA=141.7m +rank0 [2026-10-28 11:43:39,929] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:39,986] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:43:39,989] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:43:40,138] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:41,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:42,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:42,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:43,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:43,405] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:43,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:43,462] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:43:43,465] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:43,613] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:45,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:45,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:46,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:46,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:46,867] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:46,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:46,924] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:43:46,926] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:43:46,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:48,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:48,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:49,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:50,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:50,169] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:50,176] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:43:50,432] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:43:50,842] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:52,277] INFO : Plotted jets +rank0 [2026-10-28 11:43:53,357] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:53,389] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:53,619] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:54,009] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:55,842] INFO : Plotted jets +rank0 [2026-10-28 11:43:56,943] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:56,975] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 11:43:57,177] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 11:43:57,540] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:58,913] INFO : Plotted jets +rank0 [2026-10-28 11:43:59,964] INFO : Plotted jet ratio +rank0 [2026-10-28 11:44:15,633] INFO : Step 9100/50000 rank0 | Train Loss: 11.1304 | Train Loss Unweighted: 3.9345 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.4702 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2063 | Regression_eta: 0.0154 | Regression_pt: 0.2009 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9015 | Classification_binary: 3.4702 | Regression_cos_phi: 0.5643 | Regression_energy: 2.2550 | Regression_eta: 0.6881 | Regression_pt: 2.3874 | Regression_sin_phi: 0.8638 | LR: 9.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1342s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3024 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3554.0 | uss_MB: 3054.9 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:44:15,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:31,613] INFO : Step 9200/50000 rank0 | Train Loss: 11.5859 | Train Loss Unweighted: 3.9661 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2151 | Regression_eta: 0.0215 | Regression_pt: 0.2052 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9089 | Classification_binary: 3.4817 | Regression_cos_phi: 0.6610 | Regression_energy: 2.3517 | Regression_eta: 0.9569 | Regression_pt: 2.4384 | Regression_sin_phi: 0.7873 | LR: 9.27e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1005s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3416.0 | uss_MB: 2915.7 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:44:31,829] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:47,718] INFO : Step 9300/50000 rank0 | Train Loss: 10.5747 | Train Loss Unweighted: 3.8222 | Train Loss Components: Classification: 0.0227 | Classification_binary: 3.3867 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1943 | Regression_eta: 0.0128 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8538 | Classification_binary: 3.3867 | Regression_cos_phi: 0.4826 | Regression_energy: 2.1241 | Regression_eta: 0.5729 | Regression_pt: 2.2416 | Regression_sin_phi: 0.9131 | LR: 9.25e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.1399s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.2858 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3416.0 | uss_MB: 2916.8 | child_rss_MB: 3212.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:44:47,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:03,887] INFO : Step 9400/50000 rank0 | Train Loss: 11.3247 | Train Loss Unweighted: 4.0123 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.5723 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1947 | Regression_eta: 0.0220 | Regression_pt: 0.1801 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8266 | Classification_binary: 3.5723 | Regression_cos_phi: 0.7184 | Regression_energy: 2.1287 | Regression_eta: 0.9833 | Regression_pt: 2.1405 | Regression_sin_phi: 0.9549 | LR: 9.24e-04 | DataLoad Time: 0.0300s | Model Forward Time: 1.2423s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 3416.0 | uss_MB: 2917.0 | child_rss_MB: 3212.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:45:04,113] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:19,698] INFO : Step 9500/50000 rank0 | Train Loss: 9.7562 | Train Loss Unweighted: 3.7062 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.3147 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1732 | Regression_eta: 0.0092 | Regression_pt: 0.1700 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8831 | Classification_binary: 3.3147 | Regression_cos_phi: 0.5149 | Regression_energy: 1.8937 | Regression_eta: 0.4100 | Regression_pt: 2.0204 | Regression_sin_phi: 0.7193 | LR: 9.22e-04 | DataLoad Time: 0.1256s | Model Forward Time: 1.1692s | Phase Times: max_pre_step_barrier: 0.0212 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1075 | max_tensorboard_logging: 0.3119 | max_tensorboard_flush: 0.0131 | Memory: rss_MB: 3416.0 | uss_MB: 2917.2 | child_rss_MB: 3212.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:19,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:35,959] INFO : Step 9600/50000 rank0 | Train Loss: 10.0621 | Train Loss Unweighted: 3.7616 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.3592 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1770 | Regression_eta: 0.0125 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9073 | Classification_binary: 3.3592 | Regression_cos_phi: 0.5241 | Regression_energy: 1.9346 | Regression_eta: 0.5593 | Regression_pt: 2.0576 | Regression_sin_phi: 0.7201 | LR: 9.21e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2404s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.2855 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3416.0 | uss_MB: 2917.3 | child_rss_MB: 3214.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:45:36,185] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:51,589] INFO : Step 9700/50000 rank0 | Train Loss: 9.8348 | Train Loss Unweighted: 3.7436 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.3450 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1767 | Regression_eta: 0.0110 | Regression_pt: 0.1735 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8816 | Classification_binary: 3.3450 | Regression_cos_phi: 0.4312 | Regression_energy: 1.9322 | Regression_eta: 0.4891 | Regression_pt: 2.0619 | Regression_sin_phi: 0.6940 | LR: 9.19e-04 | DataLoad Time: 0.0052s | Model Forward Time: 1.2646s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0352 | max_backward: 0.0341 | max_optimizer_step: 0.1077 | max_tensorboard_logging: 0.4034 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3416.0 | uss_MB: 2916.1 | child_rss_MB: 3207.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:45:51,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:07,376] INFO : Step 9800/50000 rank0 | Train Loss: 10.4143 | Train Loss Unweighted: 3.7808 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.3655 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1841 | Regression_eta: 0.0162 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8891 | Classification_binary: 3.3655 | Regression_cos_phi: 0.6076 | Regression_energy: 2.0126 | Regression_eta: 0.7218 | Regression_pt: 2.0691 | Regression_sin_phi: 0.7486 | LR: 9.17e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2033s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0329 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.3176 | max_tensorboard_flush: 0.0148 | Memory: rss_MB: 3414.4 | uss_MB: 2916.0 | child_rss_MB: 3214.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:46:07,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:23,134] INFO : Step 9900/50000 rank0 | Train Loss: 11.4992 | Train Loss Unweighted: 3.9741 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.5054 | Regression_cos_phi: 0.0088 | Regression_energy: 0.2074 | Regression_eta: 0.0143 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9078 | Classification_binary: 3.5054 | Regression_cos_phi: 0.5713 | Regression_energy: 2.2672 | Regression_eta: 0.6392 | Regression_pt: 2.3915 | Regression_sin_phi: 1.2168 | LR: 9.16e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.3277s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0990 | max_loss: 0.0032 | max_backward: 0.0387 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.3895 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3415.4 | uss_MB: 2919.8 | child_rss_MB: 3216.3 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:46:23,348] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:38,823] INFO : Step 10000/50000 rank0 | Train Loss: 11.1498 | Train Loss Unweighted: 3.9044 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.4267 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2157 | Regression_eta: 0.0131 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8916 | Classification_binary: 3.4267 | Regression_cos_phi: 0.6378 | Regression_energy: 2.3583 | Regression_eta: 0.5847 | Regression_pt: 2.4597 | Regression_sin_phi: 0.7909 | LR: 9.14e-04 | DataLoad Time: 0.0397s | Model Forward Time: 1.1513s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0178 | max_loss: 0.0030 | max_backward: 0.0391 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3415.4 | uss_MB: 2920.4 | child_rss_MB: 3213.2 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:46:39,049] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:39,069] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 11:46:39,152] INFO : Running validation on rank0 +rank0 [2026-10-28 11:46:39,152] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:46:39,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:46:40,371] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 11:46:40,517] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:46:40,647] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:46:40,720] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:46:40,788] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:46:40,856] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:46:40,926] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:46:40,995] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:46:41,097] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:46:41,218] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:46:41,339] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:46:41,409] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:46:41,478] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:46:41,545] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:46:41,614] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:46:41,683] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:46:41,786] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:46:41,792] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:46:41,795] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:46:41,796] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:46:41,798] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:46:41,799] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:46:41,802] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:46:41,803] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:46:41,805] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:46:41,808] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:46:41,809] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:46:41,812] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:46:41,814] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:46:41,815] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:46:41,818] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:46:41,820] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:46:47,729] INFO : VALIDATION | Step=10000/50000 | Train Loss=11.1498 | Valid Loss=10.5914 | Train Loss Unweighted=3.9044 | Valid Loss Unweighted=3.8078 | Stale=0 | ETA=137.0m +rank0 [2026-10-28 11:46:47,737] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:47,793] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:46:47,797] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:46:47,945] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:49,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:50,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:50,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:51,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:51,337] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:46:51,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:51,396] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:46:51,399] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:51,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:52,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:53,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:53,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:54,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:54,707] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:54,710] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:54,765] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:46:55,040] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:46:55,188] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:56,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:57,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:57,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:58,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:58,321] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:58,328] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:46:58,588] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:46:58,991] INFO : Plotted number of elements +rank0 [2026-10-28 11:47:00,424] INFO : Plotted jets +rank0 [2026-10-28 11:47:01,501] INFO : Plotted jet ratio +rank0 [2026-10-28 11:47:01,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:47:01,763] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:47:02,154] INFO : Plotted number of elements +rank0 [2026-10-28 11:47:03,958] INFO : Plotted jets +rank0 [2026-10-28 11:47:05,061] INFO : Plotted jet ratio +rank0 [2026-10-28 11:47:05,093] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 11:47:05,297] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 11:47:05,659] INFO : Plotted number of elements +rank0 [2026-10-28 11:47:07,035] INFO : Plotted jets +rank0 [2026-10-28 11:47:08,081] INFO : Plotted jet ratio +rank0 [2026-10-28 11:47:23,759] INFO : Step 10100/50000 rank0 | Train Loss: 10.5975 | Train Loss Unweighted: 3.8344 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.4326 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1719 | Regression_eta: 0.0183 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9911 | Classification_binary: 3.4326 | Regression_cos_phi: 0.7014 | Regression_energy: 1.8798 | Regression_eta: 0.8148 | Regression_pt: 1.9727 | Regression_sin_phi: 0.8051 | LR: 9.12e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.1551s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0134 | Memory: rss_MB: 3627.7 | uss_MB: 3132.6 | child_rss_MB: 3208.0 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:47:23,975] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:39,339] INFO : Step 10200/50000 rank0 | Train Loss: 10.6366 | Train Loss Unweighted: 3.7614 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.3285 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1874 | Regression_eta: 0.0170 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9376 | Classification_binary: 3.3285 | Regression_cos_phi: 0.5613 | Regression_energy: 2.0486 | Regression_eta: 0.7587 | Regression_pt: 2.2188 | Regression_sin_phi: 0.7833 | LR: 9.11e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2613s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.3045 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3487.2 | uss_MB: 2991.2 | child_rss_MB: 3209.6 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:47:39,566] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:55,160] INFO : Step 10300/50000 rank0 | Train Loss: 11.8374 | Train Loss Unweighted: 3.9108 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.4172 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2166 | Regression_eta: 0.0249 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9414 | Classification_binary: 3.4172 | Regression_cos_phi: 0.6038 | Regression_energy: 2.3678 | Regression_eta: 1.1114 | Regression_pt: 2.4725 | Regression_sin_phi: 0.9234 | LR: 9.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1286s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1075 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3487.2 | uss_MB: 2992.0 | child_rss_MB: 3211.1 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:47:55,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:10,646] INFO : Step 10400/50000 rank0 | Train Loss: 10.2782 | Train Loss Unweighted: 3.8071 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3794 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1907 | Regression_eta: 0.0098 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9569 | Classification_binary: 3.3794 | Regression_cos_phi: 0.4522 | Regression_energy: 2.0848 | Regression_eta: 0.4363 | Regression_pt: 2.2204 | Regression_sin_phi: 0.7481 | LR: 9.07e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1127 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3487.2 | uss_MB: 2992.4 | child_rss_MB: 3215.8 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:48:10,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:25,985] INFO : Step 10500/50000 rank0 | Train Loss: 10.5189 | Train Loss Unweighted: 3.8277 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3966 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1905 | Regression_eta: 0.0120 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9568 | Classification_binary: 3.3966 | Regression_cos_phi: 0.5339 | Regression_energy: 2.0830 | Regression_eta: 0.5351 | Regression_pt: 2.2164 | Regression_sin_phi: 0.7972 | LR: 9.06e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1257s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0031 | max_backward: 0.0334 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3488.2 | uss_MB: 2993.0 | child_rss_MB: 3219.0 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:26,198] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:41,751] INFO : Step 10600/50000 rank0 | Train Loss: 11.2211 | Train Loss Unweighted: 3.8899 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.4175 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2106 | Regression_eta: 0.0170 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9227 | Classification_binary: 3.4175 | Regression_cos_phi: 0.5778 | Regression_energy: 2.3025 | Regression_eta: 0.7601 | Regression_pt: 2.4059 | Regression_sin_phi: 0.8346 | LR: 9.04e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1689s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1349 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3488.2 | uss_MB: 2993.1 | child_rss_MB: 3218.4 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:41,989] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:57,629] INFO : Step 10700/50000 rank0 | Train Loss: 11.0318 | Train Loss Unweighted: 3.8420 | Train Loss Components: Classification: 0.0233 | Classification_binary: 3.4021 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1948 | Regression_eta: 0.0204 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8778 | Classification_binary: 3.4021 | Regression_cos_phi: 0.5850 | Regression_energy: 2.1295 | Regression_eta: 0.9079 | Regression_pt: 2.1642 | Regression_sin_phi: 0.9652 | LR: 9.02e-04 | DataLoad Time: 0.0892s | Model Forward Time: 1.3055s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1084 | max_forward: 0.0139 | max_loss: 0.0035 | max_backward: 0.0332 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.3691 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3488.2 | uss_MB: 2993.1 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:48:57,843] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:13,134] INFO : Step 10800/50000 rank0 | Train Loss: 10.3468 | Train Loss Unweighted: 3.6965 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.2634 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1932 | Regression_eta: 0.0099 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9966 | Classification_binary: 3.2634 | Regression_cos_phi: 0.5483 | Regression_energy: 2.1120 | Regression_eta: 0.4398 | Regression_pt: 2.2227 | Regression_sin_phi: 0.7640 | LR: 9.00e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1744s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3300 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 3488.2 | uss_MB: 2993.2 | child_rss_MB: 3218.5 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:49:13,349] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:28,433] INFO : Step 10900/50000 rank0 | Train Loss: 9.8999 | Train Loss Unweighted: 3.7625 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.3598 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1800 | Regression_eta: 0.0105 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8417 | Classification_binary: 3.3598 | Regression_cos_phi: 0.4500 | Regression_energy: 1.9678 | Regression_eta: 0.4683 | Regression_pt: 2.0821 | Regression_sin_phi: 0.7303 | LR: 8.99e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1345s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0030 | max_backward: 0.0328 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.2894 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3488.3 | uss_MB: 2992.6 | child_rss_MB: 3217.7 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:49:28,645] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:43,755] INFO : Step 11000/50000 rank0 | Train Loss: 9.9850 | Train Loss Unweighted: 3.6916 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.2872 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1817 | Regression_eta: 0.0143 | Regression_pt: 0.1711 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8452 | Classification_binary: 3.2872 | Regression_cos_phi: 0.4527 | Regression_energy: 1.9870 | Regression_eta: 0.6369 | Regression_pt: 2.0329 | Regression_sin_phi: 0.7432 | LR: 8.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2486s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0288 | max_forward: 0.0134 | max_loss: 0.0032 | max_backward: 0.0320 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3852 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3488.3 | uss_MB: 2992.7 | child_rss_MB: 3220.0 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:49:43,967] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:44,022] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 11:49:44,104] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:44,104] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:44,104] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:49:45,199] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 11:49:45,344] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:49:45,475] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:49:45,549] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:49:45,617] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:49:45,683] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:49:45,752] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:49:45,820] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:49:45,925] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:49:46,056] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:49:46,185] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:49:46,257] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:49:46,325] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:49:46,392] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:49:46,461] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:49:46,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:49:46,634] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:49:46,641] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:49:46,643] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:49:46,645] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:49:46,647] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:49:46,650] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:49:46,651] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:49:46,653] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:49:46,656] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:49:46,657] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:49:46,659] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:49:46,662] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:49:46,663] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:49:46,665] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:49:46,666] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:49:46,669] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:49:52,714] INFO : VALIDATION | Step=11000/50000 | Train Loss=9.9850 | Valid Loss=10.3084 | Train Loss Unweighted=3.6916 | Valid Loss Unweighted=3.7173 | Stale=0 | ETA=132.3m +rank0 [2026-10-28 11:49:52,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:52,781] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:49:52,784] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:52,936] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:54,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:55,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:55,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:56,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:56,465] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:49:56,469] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:56,524] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:49:56,527] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:56,586] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:58,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:58,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:59,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:59,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:59,809] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:49:59,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:59,867] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:49:59,870] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:49:59,957] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:50:01,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:50:01,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:50:02,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:50:02,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:50:02,969] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:50:02,976] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:50:03,230] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:50:03,632] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:05,070] INFO : Plotted jets +rank0 [2026-10-28 11:50:06,143] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:06,175] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:50:06,401] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:50:06,791] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:08,602] INFO : Plotted jets +rank0 [2026-10-28 11:50:09,700] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:09,733] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:09,939] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:10,304] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:11,681] INFO : Plotted jets +rank0 [2026-10-28 11:50:12,720] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:27,959] INFO : Step 11100/50000 rank0 | Train Loss: 10.5068 | Train Loss Unweighted: 3.6474 | Train Loss Components: Classification: 0.0230 | Classification_binary: 3.2021 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1993 | Regression_eta: 0.0143 | Regression_pt: 0.1918 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8640 | Classification_binary: 3.2021 | Regression_cos_phi: 0.5654 | Regression_energy: 2.1784 | Regression_eta: 0.6369 | Regression_pt: 2.2789 | Regression_sin_phi: 0.7811 | LR: 8.95e-04 | DataLoad Time: 0.0879s | Model Forward Time: 1.2050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0662 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3696.7 | uss_MB: 3203.7 | child_rss_MB: 3233.4 | allocated_MB: 210.7 | reserved_MB: 2746.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:50:28,170] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:43,331] INFO : Step 11200/50000 rank0 | Train Loss: 10.8775 | Train Loss Unweighted: 3.7238 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.2707 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2010 | Regression_eta: 0.0194 | Regression_pt: 0.1931 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8358 | Classification_binary: 3.2707 | Regression_cos_phi: 0.5255 | Regression_energy: 2.1976 | Regression_eta: 0.8632 | Regression_pt: 2.2941 | Regression_sin_phi: 0.8905 | LR: 8.93e-04 | DataLoad Time: 0.0187s | Model Forward Time: 1.2969s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.1351 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3048 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3696.7 | uss_MB: 3201.7 | child_rss_MB: 3215.4 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:50:43,542] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:58,368] INFO : Step 11300/50000 rank0 | Train Loss: 9.8796 | Train Loss Unweighted: 3.6881 | Train Loss Components: Classification: 0.0246 | Classification_binary: 3.2907 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1756 | Regression_eta: 0.0105 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9264 | Classification_binary: 3.2907 | Regression_cos_phi: 0.5450 | Regression_energy: 1.9199 | Regression_eta: 0.4703 | Regression_pt: 2.0303 | Regression_sin_phi: 0.6969 | LR: 8.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0427 | max_forward: 0.0137 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3675 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3558.3 | uss_MB: 3064.8 | child_rss_MB: 3218.2 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:50:58,581] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:13,373] INFO : Step 11400/50000 rank0 | Train Loss: 11.3860 | Train Loss Unweighted: 3.8579 | Train Loss Components: Classification: 0.0244 | Classification_binary: 3.3546 | Regression_cos_phi: 0.0091 | Regression_energy: 0.2282 | Regression_eta: 0.0133 | Regression_pt: 0.2196 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9202 | Classification_binary: 3.3546 | Regression_cos_phi: 0.5917 | Regression_energy: 2.4945 | Regression_eta: 0.5925 | Regression_pt: 2.6091 | Regression_sin_phi: 0.8235 | LR: 8.89e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2286s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.1205 | max_optimizer_step: 0.1127 | max_tensorboard_logging: 0.2843 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3559.3 | uss_MB: 3064.8 | child_rss_MB: 3213.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:51:13,586] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:28,663] INFO : Step 11500/50000 rank0 | Train Loss: 10.6484 | Train Loss Unweighted: 3.8270 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.4011 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1854 | Regression_eta: 0.0169 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9015 | Classification_binary: 3.4011 | Regression_cos_phi: 0.5417 | Regression_energy: 2.0271 | Regression_eta: 0.7526 | Regression_pt: 2.1659 | Regression_sin_phi: 0.8585 | LR: 8.88e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2278s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3559.3 | uss_MB: 3065.2 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:51:28,886] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:44,116] INFO : Step 11600/50000 rank0 | Train Loss: 10.6840 | Train Loss Unweighted: 3.8588 | Train Loss Components: Classification: 0.0227 | Classification_binary: 3.4082 | Regression_cos_phi: 0.0080 | Regression_energy: 0.2044 | Regression_eta: 0.0126 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8544 | Classification_binary: 3.4082 | Regression_cos_phi: 0.5214 | Regression_energy: 2.2351 | Regression_eta: 0.5628 | Regression_pt: 2.3115 | Regression_sin_phi: 0.7907 | LR: 8.86e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2253s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.1393 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3559.3 | uss_MB: 3065.5 | child_rss_MB: 3217.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:51:44,328] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:59,186] INFO : Step 11700/50000 rank0 | Train Loss: 10.0076 | Train Loss Unweighted: 3.6371 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.2459 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1722 | Regression_eta: 0.0177 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8991 | Classification_binary: 3.2459 | Regression_cos_phi: 0.5693 | Regression_energy: 1.8823 | Regression_eta: 0.7916 | Regression_pt: 1.9147 | Regression_sin_phi: 0.7049 | LR: 8.84e-04 | DataLoad Time: 0.0539s | Model Forward Time: 1.3118s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0644 | max_forward: 0.0167 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.3734 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3560.3 | uss_MB: 3065.5 | child_rss_MB: 3215.5 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:51:59,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:14,360] INFO : Step 11800/50000 rank0 | Train Loss: 10.2664 | Train Loss Unweighted: 3.6090 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.1746 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1944 | Regression_eta: 0.0129 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8434 | Classification_binary: 3.1746 | Regression_cos_phi: 0.6067 | Regression_energy: 2.1255 | Regression_eta: 0.5753 | Regression_pt: 2.2320 | Regression_sin_phi: 0.7088 | LR: 8.82e-04 | DataLoad Time: 0.0267s | Model Forward Time: 1.3599s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0188 | max_loss: 0.0807 | max_backward: 0.0393 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 3560.3 | uss_MB: 3066.7 | child_rss_MB: 3213.9 | allocated_MB: 210.7 | reserved_MB: 2746.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 11:52:14,571] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:29,376] INFO : Step 11900/50000 rank0 | Train Loss: 10.2338 | Train Loss Unweighted: 3.6931 | Train Loss Components: Classification: 0.0238 | Classification_binary: 3.3072 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1657 | Regression_eta: 0.0208 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8975 | Classification_binary: 3.3072 | Regression_cos_phi: 0.5901 | Regression_energy: 1.8115 | Regression_eta: 0.9293 | Regression_pt: 1.8737 | Regression_sin_phi: 0.8245 | LR: 8.80e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2321s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3559.4 | uss_MB: 3064.9 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:52:29,598] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:44,478] INFO : Step 12000/50000 rank0 | Train Loss: 10.6141 | Train Loss Unweighted: 3.7320 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.3009 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1914 | Regression_eta: 0.0163 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8596 | Classification_binary: 3.3009 | Regression_cos_phi: 0.5432 | Regression_energy: 2.0924 | Regression_eta: 0.7288 | Regression_pt: 2.1679 | Regression_sin_phi: 0.9214 | LR: 8.78e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0888 | max_forward: 0.0139 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.1056 | max_tensorboard_logging: 0.3638 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3559.4 | uss_MB: 3065.4 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:52:44,693] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:44,781] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 11:52:44,865] INFO : Running validation on rank0 +rank0 [2026-10-28 11:52:44,866] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:52:44,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:52:45,971] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 11:52:46,116] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:52:46,247] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:52:46,320] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:52:46,388] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:52:46,455] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:52:46,523] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:52:46,592] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:52:46,694] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:52:46,825] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:52:46,954] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:52:47,026] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:52:47,095] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:52:47,162] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:52:47,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:52:47,299] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:52:47,402] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:52:47,408] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:52:47,411] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:52:47,412] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:52:47,414] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:52:47,417] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:52:47,418] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:52:47,421] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:52:47,423] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:52:47,424] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:52:47,427] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:52:47,428] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:52:47,430] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:52:47,433] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:52:47,434] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:52:47,437] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:52:53,497] INFO : VALIDATION | Step=12000/50000 | Train Loss=10.6141 | Valid Loss=10.2467 | Train Loss Unweighted=3.7320 | Valid Loss Unweighted=3.7054 | Stale=0 | ETA=127.7m +rank0 [2026-10-28 11:52:53,506] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:53,563] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:52:53,567] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:52:53,718] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:55,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:55,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:56,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:56,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:56,967] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:52:56,970] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:57,025] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:52:57,028] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:57,112] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:58,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:59,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:59,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:53:00,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:53:00,361] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:53:00,364] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:53:00,417] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:53:00,420] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:53:00,571] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:53:01,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:53:02,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:53:03,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:53:03,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:53:03,959] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:53:03,966] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:53:04,226] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:53:04,632] INFO : Plotted number of elements +rank0 [2026-10-28 11:53:06,063] INFO : Plotted jets +rank0 [2026-10-28 11:53:07,143] INFO : Plotted jet ratio +rank0 [2026-10-28 11:53:07,174] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:53:07,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:53:07,796] INFO : Plotted number of elements +rank0 [2026-10-28 11:53:09,604] INFO : Plotted jets +rank0 [2026-10-28 11:53:10,712] INFO : Plotted jet ratio +rank0 [2026-10-28 11:53:10,744] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 11:53:10,954] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 11:53:11,316] INFO : Plotted number of elements +rank0 [2026-10-28 11:53:12,689] INFO : Plotted jets +rank0 [2026-10-28 11:53:13,735] INFO : Plotted jet ratio +rank0 [2026-10-28 11:53:28,716] INFO : Step 12100/50000 rank0 | Train Loss: 10.0259 | Train Loss Unweighted: 3.6415 | Train Loss Components: Classification: 0.0223 | Classification_binary: 3.2664 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1589 | Regression_eta: 0.0154 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8393 | Classification_binary: 3.2664 | Regression_cos_phi: 0.7729 | Regression_energy: 1.7370 | Regression_eta: 0.6847 | Regression_pt: 1.8731 | Regression_sin_phi: 0.8524 | LR: 8.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.1225 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.2792 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3755.4 | uss_MB: 3263.0 | child_rss_MB: 3210.7 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:28,928] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:44,007] INFO : Step 12200/50000 rank0 | Train Loss: 10.5149 | Train Loss Unweighted: 3.6655 | Train Loss Components: Classification: 0.0210 | Classification_binary: 3.2436 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1875 | Regression_eta: 0.0227 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7895 | Classification_binary: 3.2436 | Regression_cos_phi: 0.4939 | Regression_energy: 2.0499 | Regression_eta: 1.0121 | Regression_pt: 2.0678 | Regression_sin_phi: 0.8579 | LR: 8.74e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2172s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3755.4 | uss_MB: 3263.0 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:44,231] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:58,771] INFO : Step 12300/50000 rank0 | Train Loss: 10.1388 | Train Loss Unweighted: 3.6154 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.2029 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1822 | Regression_eta: 0.0128 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9409 | Classification_binary: 3.2029 | Regression_cos_phi: 0.4710 | Regression_energy: 1.9914 | Regression_eta: 0.5719 | Regression_pt: 2.0924 | Regression_sin_phi: 0.8683 | LR: 8.72e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2529s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0169 | max_loss: 0.0039 | max_backward: 0.1459 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2839 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3634.2 | uss_MB: 3144.9 | child_rss_MB: 3210.8 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:53:58,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:14,067] INFO : Step 12400/50000 rank0 | Train Loss: 10.0482 | Train Loss Unweighted: 3.6144 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.2086 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1781 | Regression_eta: 0.0153 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9443 | Classification_binary: 3.2086 | Regression_cos_phi: 0.5221 | Regression_energy: 1.9472 | Regression_eta: 0.6831 | Regression_pt: 2.0423 | Regression_sin_phi: 0.7008 | LR: 8.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2463s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0033 | max_backward: 0.0381 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3634.2 | uss_MB: 3145.3 | child_rss_MB: 3208.6 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:54:14,291] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:29,113] INFO : Step 12500/50000 rank0 | Train Loss: 10.0764 | Train Loss Unweighted: 3.6261 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.1989 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1906 | Regression_eta: 0.0105 | Regression_pt: 0.1880 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8570 | Classification_binary: 3.1989 | Regression_cos_phi: 0.4602 | Regression_energy: 2.0839 | Regression_eta: 0.4688 | Regression_pt: 2.2334 | Regression_sin_phi: 0.7742 | LR: 8.68e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0481 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1112 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3634.2 | uss_MB: 3143.8 | child_rss_MB: 3207.2 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:54:29,324] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:43,996] INFO : Step 12600/50000 rank0 | Train Loss: 9.7628 | Train Loss Unweighted: 3.6329 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.2504 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1649 | Regression_eta: 0.0133 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9003 | Classification_binary: 3.2504 | Regression_cos_phi: 0.5113 | Regression_energy: 1.8030 | Regression_eta: 0.5925 | Regression_pt: 1.9563 | Regression_sin_phi: 0.7491 | LR: 8.66e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1328s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.2893 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3634.2 | uss_MB: 3143.9 | child_rss_MB: 3207.2 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:44,211] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:58,831] INFO : Step 12700/50000 rank0 | Train Loss: 9.7425 | Train Loss Unweighted: 3.5927 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.2049 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1699 | Regression_eta: 0.0137 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9389 | Classification_binary: 3.2049 | Regression_cos_phi: 0.4641 | Regression_energy: 1.8576 | Regression_eta: 0.6103 | Regression_pt: 1.9548 | Regression_sin_phi: 0.7118 | LR: 8.64e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2236s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0159 | max_loss: 0.0030 | max_backward: 0.0338 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3634.2 | uss_MB: 3144.0 | child_rss_MB: 3205.1 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:59,063] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:13,503] INFO : Step 12800/50000 rank0 | Train Loss: 10.2311 | Train Loss Unweighted: 3.5411 | Train Loss Components: Classification: 0.0240 | Classification_binary: 3.1167 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1891 | Regression_eta: 0.0132 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9018 | Classification_binary: 3.1167 | Regression_cos_phi: 0.6234 | Regression_energy: 2.0669 | Regression_eta: 0.5880 | Regression_pt: 2.1415 | Regression_sin_phi: 0.7929 | LR: 8.62e-04 | DataLoad Time: 0.0097s | Model Forward Time: 1.1792s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0135 | max_loss: 0.0030 | max_backward: 0.0331 | max_optimizer_step: 0.1449 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3634.2 | uss_MB: 3144.2 | child_rss_MB: 3203.9 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:55:13,732] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:28,261] INFO : Step 12900/50000 rank0 | Train Loss: 10.0436 | Train Loss Unweighted: 3.5462 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.1463 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1735 | Regression_eta: 0.0159 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8863 | Classification_binary: 3.1463 | Regression_cos_phi: 0.5189 | Regression_energy: 1.8970 | Regression_eta: 0.7095 | Regression_pt: 2.0172 | Regression_sin_phi: 0.8684 | LR: 8.60e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2310s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0035 | max_backward: 0.1413 | max_optimizer_step: 0.1078 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3632.8 | uss_MB: 3142.9 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:55:28,473] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:43,264] INFO : Step 13000/50000 rank0 | Train Loss: 9.9116 | Train Loss Unweighted: 3.5848 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.1932 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1730 | Regression_eta: 0.0129 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8357 | Classification_binary: 3.1932 | Regression_cos_phi: 0.4354 | Regression_energy: 1.8913 | Regression_eta: 0.5767 | Regression_pt: 1.9745 | Regression_sin_phi: 1.0048 | LR: 8.58e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1122 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3632.8 | uss_MB: 3145.9 | child_rss_MB: 3206.5 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:55:43,476] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:43,494] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 11:55:43,684] INFO : Running validation on rank0 +rank0 [2026-10-28 11:55:43,684] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:55:43,684] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:55:44,767] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 11:55:44,911] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:55:45,042] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:55:45,116] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:55:45,184] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:55:45,251] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:55:45,320] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:55:45,389] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:55:45,496] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:55:45,626] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:55:45,756] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:55:45,826] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:55:45,894] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:55:45,962] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:55:46,031] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:55:46,099] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:55:46,203] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:55:46,209] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:55:46,212] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:55:46,214] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:55:46,215] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:55:46,218] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:55:46,221] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:55:46,222] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:55:46,224] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:55:46,226] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:55:46,228] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:55:46,230] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:55:46,231] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:55:46,234] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:55:46,235] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:55:46,237] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:55:52,088] INFO : VALIDATION | Step=13000/50000 | Train Loss=9.9116 | Valid Loss=10.0401 | Train Loss Unweighted=3.5848 | Valid Loss Unweighted=3.6257 | Stale=0 | ETA=123.3m +rank0 [2026-10-28 11:55:52,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:52,153] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:55:52,157] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:55:52,305] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:53,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:54,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:55,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:55,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:55,882] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:55:55,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:55,940] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:55:55,943] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:56,026] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:57,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:57,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:58,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:59,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:59,123] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:55:59,126] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:59,180] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:55:59,183] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:55:59,268] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:00,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:01,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:01,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:02,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:02,521] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:56:02,529] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:56:02,783] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:56:03,189] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:04,628] INFO : Plotted jets +rank0 [2026-10-28 11:56:05,709] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:05,741] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:05,970] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:06,361] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:08,187] INFO : Plotted jets +rank0 [2026-10-28 11:56:09,289] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:09,322] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 11:56:09,527] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 11:56:09,892] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:11,273] INFO : Plotted jets +rank0 [2026-10-28 11:56:12,319] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:27,259] INFO : Step 13100/50000 rank0 | Train Loss: 10.3707 | Train Loss Unweighted: 3.5507 | Train Loss Components: Classification: 0.0238 | Classification_binary: 3.1203 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1893 | Regression_eta: 0.0158 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8954 | Classification_binary: 3.1203 | Regression_cos_phi: 0.5779 | Regression_energy: 2.0694 | Regression_eta: 0.7032 | Regression_pt: 2.1869 | Regression_sin_phi: 0.8177 | LR: 8.56e-04 | DataLoad Time: 0.0198s | Model Forward Time: 1.3487s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1045 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3826.7 | uss_MB: 3338.2 | child_rss_MB: 3209.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:56:27,471] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:42,171] INFO : Step 13200/50000 rank0 | Train Loss: 11.3362 | Train Loss Unweighted: 3.6603 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.1853 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2119 | Regression_eta: 0.0237 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8331 | Classification_binary: 3.1853 | Regression_cos_phi: 0.6062 | Regression_energy: 2.3167 | Regression_eta: 1.0552 | Regression_pt: 2.3466 | Regression_sin_phi: 0.9931 | LR: 8.54e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2562s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.2871 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3827.7 | uss_MB: 3338.2 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:56:42,402] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:56,883] INFO : Step 13300/50000 rank0 | Train Loss: 10.1262 | Train Loss Unweighted: 3.5564 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.1313 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1878 | Regression_eta: 0.0132 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8179 | Classification_binary: 3.1313 | Regression_cos_phi: 0.5104 | Regression_energy: 2.0528 | Regression_eta: 0.5870 | Regression_pt: 2.2096 | Regression_sin_phi: 0.8172 | LR: 8.52e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2289s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0165 | max_loss: 0.0032 | max_backward: 0.0385 | max_optimizer_step: 0.1625 | max_tensorboard_logging: 0.2998 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3706.2 | uss_MB: 3218.1 | child_rss_MB: 3206.9 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:56:57,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:11,579] INFO : Step 13400/50000 rank0 | Train Loss: 9.4509 | Train Loss Unweighted: 3.6335 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.2492 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1719 | Regression_eta: 0.0107 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7766 | Classification_binary: 3.2492 | Regression_cos_phi: 0.4707 | Regression_energy: 1.8788 | Regression_eta: 0.4757 | Regression_pt: 1.9903 | Regression_sin_phi: 0.6097 | LR: 8.50e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2559s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0030 | max_backward: 0.0329 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3706.2 | uss_MB: 3217.1 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:57:11,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:26,275] INFO : Step 13500/50000 rank0 | Train Loss: 10.9049 | Train Loss Unweighted: 3.7137 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.2672 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1980 | Regression_eta: 0.0168 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9430 | Classification_binary: 3.2672 | Regression_cos_phi: 0.6029 | Regression_energy: 2.1651 | Regression_eta: 0.7471 | Regression_pt: 2.2252 | Regression_sin_phi: 0.9545 | LR: 8.48e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0340 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3706.2 | uss_MB: 3217.3 | child_rss_MB: 3206.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:26,502] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:41,007] INFO : Step 13600/50000 rank0 | Train Loss: 9.0932 | Train Loss Unweighted: 3.4680 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0927 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1663 | Regression_eta: 0.0100 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8342 | Classification_binary: 3.0927 | Regression_cos_phi: 0.4130 | Regression_energy: 1.8176 | Regression_eta: 0.4481 | Regression_pt: 1.9600 | Regression_sin_phi: 0.5275 | LR: 8.45e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1732s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0171 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1297 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3707.2 | uss_MB: 3220.0 | child_rss_MB: 3204.2 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:57:41,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:55,711] INFO : Step 13700/50000 rank0 | Train Loss: 9.4003 | Train Loss Unweighted: 3.4982 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.1359 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1585 | Regression_eta: 0.0121 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8471 | Classification_binary: 3.1359 | Regression_cos_phi: 0.5673 | Regression_energy: 1.7323 | Regression_eta: 0.5405 | Regression_pt: 1.8106 | Regression_sin_phi: 0.7665 | LR: 8.43e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2686s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0166 | max_loss: 0.0032 | max_backward: 0.0379 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3708.2 | uss_MB: 3220.3 | child_rss_MB: 3203.9 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:57:55,944] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:10,411] INFO : Step 13800/50000 rank0 | Train Loss: 9.7937 | Train Loss Unweighted: 3.6255 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.2366 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1707 | Regression_eta: 0.0122 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9230 | Classification_binary: 3.2366 | Regression_cos_phi: 0.5119 | Regression_energy: 1.8657 | Regression_eta: 0.5426 | Regression_pt: 1.9707 | Regression_sin_phi: 0.7432 | LR: 8.41e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2280s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.2159 | max_tensorboard_logging: 0.2767 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3708.2 | uss_MB: 3218.4 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:58:10,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:24,749] INFO : Step 13900/50000 rank0 | Train Loss: 10.1377 | Train Loss Unweighted: 3.6079 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.1934 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1858 | Regression_eta: 0.0142 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8808 | Classification_binary: 3.1934 | Regression_cos_phi: 0.5914 | Regression_energy: 2.0315 | Regression_eta: 0.6342 | Regression_pt: 2.0695 | Regression_sin_phi: 0.7370 | LR: 8.39e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0841 | max_loss: 0.0031 | max_backward: 0.0337 | max_optimizer_step: 0.1115 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3706.4 | uss_MB: 3216.3 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:58:24,962] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:39,510] INFO : Step 14000/50000 rank0 | Train Loss: 9.9364 | Train Loss Unweighted: 3.4252 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0045 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1892 | Regression_eta: 0.0128 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8269 | Classification_binary: 3.0045 | Regression_cos_phi: 0.4507 | Regression_energy: 2.0689 | Regression_eta: 0.5692 | Regression_pt: 2.1461 | Regression_sin_phi: 0.8701 | LR: 8.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2662s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0566 | max_forward: 0.0170 | max_loss: 0.0032 | max_backward: 0.0383 | max_optimizer_step: 0.1085 | max_tensorboard_logging: 0.3506 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3707.4 | uss_MB: 3220.0 | child_rss_MB: 3206.1 | allocated_MB: 210.7 | reserved_MB: 2770.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 11:58:39,721] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:58:39,794] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 11:58:39,877] INFO : Running validation on rank0 +rank0 [2026-10-28 11:58:39,877] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:58:39,877] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:58:40,958] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 11:58:41,104] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:58:41,236] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:58:41,310] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:58:41,379] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:58:41,446] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:58:41,516] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:58:41,585] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:58:41,691] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:58:41,813] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:58:41,935] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:58:42,008] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:58:42,077] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:58:42,145] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:58:42,214] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:58:42,283] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:58:42,391] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:58:42,398] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:58:42,401] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:58:42,404] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:58:42,406] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:58:42,407] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:58:42,410] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:58:42,411] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:58:42,414] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:58:42,416] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:58:42,418] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:58:42,421] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:58:42,424] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:58:42,427] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:58:42,430] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:58:42,433] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:58:48,245] INFO : VALIDATION | Step=14000/50000 | Train Loss=9.9364 | Valid Loss=9.9343 | Train Loss Unweighted=3.4252 | Valid Loss Unweighted=3.5593 | Stale=0 | ETA=118.9m +rank0 [2026-10-28 11:58:48,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:48,310] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:58:48,314] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:58:48,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:49,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:50,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:51,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:51,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:51,691] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:51,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:51,750] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:58:51,753] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:51,840] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:53,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:53,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:54,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:54,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:55,083] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:55,086] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:55,138] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:58:55,141] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:58:55,231] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:56,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:57,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:57,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:58,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:58,397] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:58,404] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:58:58,664] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:58:59,072] INFO : Plotted number of elements +rank0 [2026-10-28 11:59:00,520] INFO : Plotted jets +rank0 [2026-10-28 11:59:01,594] INFO : Plotted jet ratio +rank0 [2026-10-28 11:59:01,625] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:59:01,856] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:59:02,247] INFO : Plotted number of elements +rank0 [2026-10-28 11:59:04,055] INFO : Plotted jets +rank0 [2026-10-28 11:59:05,163] INFO : Plotted jet ratio +rank0 [2026-10-28 11:59:05,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 11:59:05,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 11:59:05,764] INFO : Plotted number of elements +rank0 [2026-10-28 11:59:07,146] INFO : Plotted jets +rank0 [2026-10-28 11:59:08,192] INFO : Plotted jet ratio +rank0 [2026-10-28 11:59:23,028] INFO : Step 14100/50000 rank0 | Train Loss: 9.3133 | Train Loss Unweighted: 3.4521 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.0713 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1691 | Regression_eta: 0.0102 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8477 | Classification_binary: 3.0713 | Regression_cos_phi: 0.4207 | Regression_energy: 1.8487 | Regression_eta: 0.4531 | Regression_pt: 1.9601 | Regression_sin_phi: 0.7117 | LR: 8.35e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1134 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3885 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3922.4 | uss_MB: 3434.0 | child_rss_MB: 3206.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:59:23,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:37,664] INFO : Step 14200/50000 rank0 | Train Loss: 10.1568 | Train Loss Unweighted: 3.3915 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9953 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1711 | Regression_eta: 0.0251 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8071 | Classification_binary: 2.9953 | Regression_cos_phi: 0.6147 | Regression_energy: 1.8709 | Regression_eta: 1.1188 | Regression_pt: 1.9021 | Regression_sin_phi: 0.8478 | LR: 8.32e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2841s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0330 | max_backward: 0.0329 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3803 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3922.4 | uss_MB: 3432.9 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:59:37,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:52,161] INFO : Step 14300/50000 rank0 | Train Loss: 9.1070 | Train Loss Unweighted: 3.4337 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.0592 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1669 | Regression_eta: 0.0096 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8477 | Classification_binary: 3.0592 | Regression_cos_phi: 0.4087 | Regression_energy: 1.8248 | Regression_eta: 0.4265 | Regression_pt: 1.9345 | Regression_sin_phi: 0.6055 | LR: 8.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2465s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0034 | max_backward: 0.0985 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.3079 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3779.5 | uss_MB: 3291.9 | child_rss_MB: 3227.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:59:52,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:06,735] INFO : Step 14400/50000 rank0 | Train Loss: 10.2774 | Train Loss Unweighted: 3.5045 | Train Loss Components: Classification: 0.0229 | Classification_binary: 3.0739 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1949 | Regression_eta: 0.0188 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8608 | Classification_binary: 3.0739 | Regression_cos_phi: 0.4998 | Regression_energy: 2.1304 | Regression_eta: 0.8391 | Regression_pt: 2.1201 | Regression_sin_phi: 0.7533 | LR: 8.28e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3779.5 | uss_MB: 3292.9 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:06,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:21,399] INFO : Step 14500/50000 rank0 | Train Loss: 9.7857 | Train Loss Unweighted: 3.5340 | Train Loss Components: Classification: 0.0219 | Classification_binary: 3.1233 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1812 | Regression_eta: 0.0123 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8253 | Classification_binary: 3.1233 | Regression_cos_phi: 0.4008 | Regression_energy: 1.9814 | Regression_eta: 0.5498 | Regression_pt: 2.1512 | Regression_sin_phi: 0.7540 | LR: 8.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2554s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0141 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3779.5 | uss_MB: 3293.0 | child_rss_MB: 3200.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:00:21,631] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:35,473] INFO : Step 14600/50000 rank0 | Train Loss: 9.7258 | Train Loss Unweighted: 3.5714 | Train Loss Components: Classification: 0.0218 | Classification_binary: 3.1660 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1832 | Regression_eta: 0.0133 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8206 | Classification_binary: 3.1660 | Regression_cos_phi: 0.4135 | Regression_energy: 2.0026 | Regression_eta: 0.5954 | Regression_pt: 2.0641 | Regression_sin_phi: 0.6637 | LR: 8.24e-04 | DataLoad Time: 0.0337s | Model Forward Time: 1.2034s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0035 | max_backward: 0.0763 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.2930 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3779.5 | uss_MB: 3293.0 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:35,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:49,731] INFO : Step 14700/50000 rank0 | Train Loss: 10.0717 | Train Loss Unweighted: 3.5380 | Train Loss Components: Classification: 0.0216 | Classification_binary: 3.1121 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1897 | Regression_eta: 0.0125 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8113 | Classification_binary: 3.1121 | Regression_cos_phi: 0.5481 | Regression_energy: 2.0739 | Regression_eta: 0.5564 | Regression_pt: 2.2071 | Regression_sin_phi: 0.7628 | LR: 8.21e-04 | DataLoad Time: 0.0290s | Model Forward Time: 1.2946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0561 | max_loss: 0.0034 | max_backward: 0.0399 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3798 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3779.5 | uss_MB: 3295.9 | child_rss_MB: 3208.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:00:49,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:04,051] INFO : Step 14800/50000 rank0 | Train Loss: 9.6984 | Train Loss Unweighted: 3.5725 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.1954 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1649 | Regression_eta: 0.0143 | Regression_pt: 0.1592 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8370 | Classification_binary: 3.1954 | Regression_cos_phi: 0.5023 | Regression_energy: 1.8031 | Regression_eta: 0.6381 | Regression_pt: 1.8914 | Regression_sin_phi: 0.8310 | LR: 8.19e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2661s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0169 | max_loss: 0.0033 | max_backward: 0.0383 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.2861 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3779.5 | uss_MB: 3295.9 | child_rss_MB: 3205.7 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:01:04,279] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:18,465] INFO : Step 14900/50000 rank0 | Train Loss: 10.0613 | Train Loss Unweighted: 3.5741 | Train Loss Components: Classification: 0.0226 | Classification_binary: 3.1498 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1886 | Regression_eta: 0.0115 | Regression_pt: 0.1854 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8519 | Classification_binary: 3.1498 | Regression_cos_phi: 0.5143 | Regression_energy: 2.0619 | Regression_eta: 0.5146 | Regression_pt: 2.2034 | Regression_sin_phi: 0.7653 | LR: 8.17e-04 | DataLoad Time: 0.0340s | Model Forward Time: 1.2802s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0535 | max_forward: 0.0154 | max_loss: 0.0036 | max_backward: 0.0335 | max_optimizer_step: 0.1121 | max_tensorboard_logging: 0.3629 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3778.3 | uss_MB: 3292.1 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:01:18,679] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:32,488] INFO : Step 15000/50000 rank0 | Train Loss: 9.8664 | Train Loss Unweighted: 3.3728 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.9815 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1672 | Regression_eta: 0.0190 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9226 | Classification_binary: 2.9815 | Regression_cos_phi: 0.5356 | Regression_energy: 1.8278 | Regression_eta: 0.8455 | Regression_pt: 1.9471 | Regression_sin_phi: 0.8062 | LR: 8.15e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.2739 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3778.3 | uss_MB: 3293.3 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:01:32,700] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:32,750] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 12:01:32,832] INFO : Running validation on rank0 +rank0 [2026-10-28 12:01:32,832] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:01:32,833] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:01:33,913] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 12:01:34,058] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:01:34,190] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:01:34,264] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:01:34,333] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:01:34,401] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:01:34,470] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:01:34,538] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:01:34,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:01:34,778] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:01:34,909] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:01:34,980] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:01:35,050] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:01:35,119] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:01:35,187] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:01:35,257] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:01:35,367] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:01:35,374] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:01:35,377] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:01:35,380] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:01:35,381] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:01:35,383] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:01:35,384] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:01:35,387] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:01:35,390] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:01:35,393] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:01:35,394] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:01:35,396] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:01:35,399] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:01:35,400] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:01:35,402] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:01:35,403] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:01:41,173] INFO : VALIDATION | Step=15000/50000 | Train Loss=9.8664 | Valid Loss=9.7173 | Train Loss Unweighted=3.3728 | Valid Loss Unweighted=3.5113 | Stale=0 | ETA=114.6m +rank0 [2026-10-28 12:01:41,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:41,240] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:01:41,243] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:01:41,399] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:42,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:43,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:44,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:44,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:44,739] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:01:44,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:44,797] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:01:44,800] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:44,891] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:46,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:46,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:47,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:48,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:48,140] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:48,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:48,197] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:01:48,200] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:01:48,356] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:49,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:50,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:50,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:51,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:51,417] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:51,423] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:01:51,675] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:01:52,083] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:53,524] INFO : Plotted jets +rank0 [2026-10-28 12:01:54,598] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:54,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:54,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:55,249] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:57,148] INFO : Plotted jets +rank0 [2026-10-28 12:01:58,252] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:58,284] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 12:01:58,487] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 12:01:58,850] INFO : Plotted number of elements +rank0 [2026-10-28 12:02:00,229] INFO : Plotted jets +rank0 [2026-10-28 12:02:01,274] INFO : Plotted jet ratio +rank0 [2026-10-28 12:02:15,634] INFO : Step 15100/50000 rank0 | Train Loss: 9.4367 | Train Loss Unweighted: 3.5540 | Train Loss Components: Classification: 0.0219 | Classification_binary: 3.1651 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1744 | Regression_eta: 0.0114 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8249 | Classification_binary: 3.1651 | Regression_cos_phi: 0.4123 | Regression_energy: 1.9072 | Regression_eta: 0.5085 | Regression_pt: 1.9991 | Regression_sin_phi: 0.6196 | LR: 8.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 1.2644s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.1275 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.2815 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3985.0 | uss_MB: 3501.5 | child_rss_MB: 3203.1 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:02:15,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:29,912] INFO : Step 15200/50000 rank0 | Train Loss: 9.5824 | Train Loss Unweighted: 3.4403 | Train Loss Components: Classification: 0.0233 | Classification_binary: 3.0462 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1740 | Regression_eta: 0.0130 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8775 | Classification_binary: 3.0462 | Regression_cos_phi: 0.4890 | Regression_energy: 1.9023 | Regression_eta: 0.5786 | Regression_pt: 2.0088 | Regression_sin_phi: 0.6800 | LR: 8.10e-04 | DataLoad Time: 0.1018s | Model Forward Time: 1.3048s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0015 | max_forward: 0.0246 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3985.0 | uss_MB: 3501.6 | child_rss_MB: 3205.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:02:30,127] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:44,060] INFO : Step 15300/50000 rank0 | Train Loss: 9.0698 | Train Loss Unweighted: 3.4121 | Train Loss Components: Classification: 0.0209 | Classification_binary: 3.0523 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1571 | Regression_eta: 0.0126 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7866 | Classification_binary: 3.0523 | Regression_cos_phi: 0.4914 | Regression_energy: 1.7175 | Regression_eta: 0.5602 | Regression_pt: 1.8436 | Regression_sin_phi: 0.6182 | LR: 8.08e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.1927s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3985.0 | uss_MB: 3501.6 | child_rss_MB: 3205.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:02:44,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:58,215] INFO : Step 15400/50000 rank0 | Train Loss: 10.5317 | Train Loss Unweighted: 3.6995 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.2466 | Regression_cos_phi: 0.0088 | Regression_energy: 0.2042 | Regression_eta: 0.0131 | Regression_pt: 0.1968 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8459 | Classification_binary: 3.2466 | Regression_cos_phi: 0.5710 | Regression_energy: 2.2327 | Regression_eta: 0.5838 | Regression_pt: 2.3379 | Regression_sin_phi: 0.7137 | LR: 8.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2671s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0139 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3849.0 | uss_MB: 3366.0 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:02:58,442] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:12,287] INFO : Step 15500/50000 rank0 | Train Loss: 9.0509 | Train Loss Unweighted: 3.3934 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0414 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1540 | Regression_eta: 0.0130 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8279 | Classification_binary: 3.0414 | Regression_cos_phi: 0.4716 | Regression_energy: 1.6836 | Regression_eta: 0.5809 | Regression_pt: 1.7655 | Regression_sin_phi: 0.6800 | LR: 8.03e-04 | DataLoad Time: 0.0254s | Model Forward Time: 1.2363s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.1171 | max_optimizer_step: 0.1145 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3849.0 | uss_MB: 3366.1 | child_rss_MB: 3205.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:03:12,499] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:26,208] INFO : Step 15600/50000 rank0 | Train Loss: 9.6502 | Train Loss Unweighted: 3.4492 | Train Loss Components: Classification: 0.0218 | Classification_binary: 3.0612 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1706 | Regression_eta: 0.0122 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8219 | Classification_binary: 3.0612 | Regression_cos_phi: 0.7167 | Regression_energy: 1.8651 | Regression_eta: 0.5448 | Regression_pt: 1.9622 | Regression_sin_phi: 0.6783 | LR: 8.01e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2828s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0171 | max_loss: 0.0034 | max_backward: 0.0391 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3850.0 | uss_MB: 3368.8 | child_rss_MB: 3208.0 | allocated_MB: 210.7 | reserved_MB: 2838.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:03:26,435] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:40,215] INFO : Step 15700/50000 rank0 | Train Loss: 9.9752 | Train Loss Unweighted: 3.4213 | Train Loss Components: Classification: 0.0207 | Classification_binary: 3.0207 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1793 | Regression_eta: 0.0217 | Regression_pt: 0.1626 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7800 | Classification_binary: 3.0207 | Regression_cos_phi: 0.4969 | Regression_energy: 1.9599 | Regression_eta: 0.9682 | Regression_pt: 1.9323 | Regression_sin_phi: 0.8171 | LR: 7.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2586s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3850.0 | uss_MB: 3366.8 | child_rss_MB: 3204.2 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:40,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:54,203] INFO : Step 15800/50000 rank0 | Train Loss: 9.8566 | Train Loss Unweighted: 3.5601 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.1499 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1859 | Regression_eta: 0.0125 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8302 | Classification_binary: 3.1499 | Regression_cos_phi: 0.5265 | Regression_energy: 2.0325 | Regression_eta: 0.5583 | Regression_pt: 2.0722 | Regression_sin_phi: 0.6871 | LR: 7.96e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3850.0 | uss_MB: 3366.8 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:03:54,428] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:08,179] INFO : Step 15900/50000 rank0 | Train Loss: 9.4639 | Train Loss Unweighted: 3.4243 | Train Loss Components: Classification: 0.0215 | Classification_binary: 3.0460 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1649 | Regression_eta: 0.0125 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8089 | Classification_binary: 3.0460 | Regression_cos_phi: 0.5746 | Regression_energy: 1.8025 | Regression_eta: 0.5588 | Regression_pt: 1.9338 | Regression_sin_phi: 0.7393 | LR: 7.94e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2879s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0038 | max_backward: 0.0339 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3850.0 | uss_MB: 3367.2 | child_rss_MB: 3204.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:04:08,407] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:21,950] INFO : Step 16000/50000 rank0 | Train Loss: 9.0867 | Train Loss Unweighted: 3.3120 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9470 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1604 | Regression_eta: 0.0115 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8167 | Classification_binary: 2.9470 | Regression_cos_phi: 0.5495 | Regression_energy: 1.7532 | Regression_eta: 0.5117 | Regression_pt: 1.8554 | Regression_sin_phi: 0.6533 | LR: 7.91e-04 | DataLoad Time: 0.1161s | Model Forward Time: 1.1871s | Phase Times: max_pre_step_barrier: 0.0433 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1121 | max_tensorboard_logging: 0.2842 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3850.6 | uss_MB: 3366.2 | child_rss_MB: 3208.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:04:22,162] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:22,206] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 12:04:22,289] INFO : Running validation on rank0 +rank0 [2026-10-28 12:04:22,289] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:04:22,289] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:04:23,373] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 12:04:23,519] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:04:23,651] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:04:23,725] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:04:23,794] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:04:23,861] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:04:23,931] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:04:24,002] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:04:24,109] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:04:24,241] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:04:24,373] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:04:24,444] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:04:24,513] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:04:24,582] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:04:24,652] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:04:24,722] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:04:24,832] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:04:24,839] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:04:24,842] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:04:24,844] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:04:24,845] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:04:24,848] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:04:24,852] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:04:24,854] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:04:24,855] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:04:24,858] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:04:24,859] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:04:24,861] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:04:24,864] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:04:24,865] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:04:24,867] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:04:24,868] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:04:30,699] INFO : VALIDATION | Step=16000/50000 | Train Loss=9.0867 | Valid Loss=9.6727 | Train Loss Unweighted=3.3120 | Valid Loss Unweighted=3.4797 | Stale=0 | ETA=110.4m +rank0 [2026-10-28 12:04:30,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:30,766] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:04:30,769] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:30,919] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:32,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:33,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:33,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:34,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:34,630] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:04:34,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:34,689] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:04:34,692] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:34,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:36,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:36,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:37,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:38,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:38,140] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:04:38,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:38,196] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:04:38,210] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:04:38,371] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:39,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:40,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:40,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:41,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:41,437] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:04:41,444] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:41,701] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:42,104] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:43,548] INFO : Plotted jets +rank0 [2026-10-28 12:04:44,625] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:44,657] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:44,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:45,277] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:47,133] INFO : Plotted jets +rank0 [2026-10-28 12:04:48,236] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:48,267] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:48,472] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:48,835] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:50,217] INFO : Plotted jets +rank0 [2026-10-28 12:04:51,261] INFO : Plotted jet ratio +rank0 [2026-10-28 12:05:05,455] INFO : Step 16100/50000 rank0 | Train Loss: 10.4744 | Train Loss Unweighted: 3.5254 | Train Loss Components: Classification: 0.0226 | Classification_binary: 3.0642 | Regression_cos_phi: 0.0068 | Regression_energy: 0.2056 | Regression_eta: 0.0191 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8505 | Classification_binary: 3.0642 | Regression_cos_phi: 0.4426 | Regression_energy: 2.2474 | Regression_eta: 0.8539 | Regression_pt: 2.3802 | Regression_sin_phi: 0.6356 | LR: 7.89e-04 | DataLoad Time: 0.0116s | Model Forward Time: 1.3152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1045 | max_forward: 0.0149 | max_loss: 0.0036 | max_backward: 0.0338 | max_optimizer_step: 0.1124 | max_tensorboard_logging: 0.3412 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4054.2 | uss_MB: 3570.7 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:05:05,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:19,271] INFO : Step 16200/50000 rank0 | Train Loss: 9.4108 | Train Loss Unweighted: 3.4187 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.0328 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1710 | Regression_eta: 0.0104 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8706 | Classification_binary: 3.0328 | Regression_cos_phi: 0.5153 | Regression_energy: 1.8692 | Regression_eta: 0.4645 | Regression_pt: 1.9743 | Regression_sin_phi: 0.6841 | LR: 7.86e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3032s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0612 | max_backward: 0.0343 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3689 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 4054.2 | uss_MB: 3569.5 | child_rss_MB: 3209.7 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:05:19,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:33,096] INFO : Step 16300/50000 rank0 | Train Loss: 9.4145 | Train Loss Unweighted: 3.4893 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.1185 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1627 | Regression_eta: 0.0138 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8461 | Classification_binary: 3.1185 | Regression_cos_phi: 0.5067 | Regression_energy: 1.7787 | Regression_eta: 0.6168 | Regression_pt: 1.8625 | Regression_sin_phi: 0.6852 | LR: 7.84e-04 | DataLoad Time: 0.0262s | Model Forward Time: 1.2520s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0159 | max_loss: 0.0032 | max_backward: 0.0901 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4054.2 | uss_MB: 3570.8 | child_rss_MB: 3207.2 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:05:33,310] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:46,983] INFO : Step 16400/50000 rank0 | Train Loss: 9.6569 | Train Loss Unweighted: 3.3684 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9649 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1808 | Regression_eta: 0.0150 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7963 | Classification_binary: 2.9649 | Regression_cos_phi: 0.4123 | Regression_energy: 1.9771 | Regression_eta: 0.6709 | Regression_pt: 2.0400 | Regression_sin_phi: 0.7953 | LR: 7.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2587s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.2783 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3925.7 | uss_MB: 3440.3 | child_rss_MB: 3211.0 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:05:47,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:00,994] INFO : Step 16500/50000 rank0 | Train Loss: 9.9381 | Train Loss Unweighted: 3.3857 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.9655 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1862 | Regression_eta: 0.0136 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8481 | Classification_binary: 2.9655 | Regression_cos_phi: 0.5776 | Regression_energy: 2.0353 | Regression_eta: 0.6047 | Regression_pt: 2.1522 | Regression_sin_phi: 0.7547 | LR: 7.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2145s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0035 | max_backward: 0.0872 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3925.7 | uss_MB: 3440.4 | child_rss_MB: 3206.6 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:06:01,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:15,243] INFO : Step 16600/50000 rank0 | Train Loss: 9.7441 | Train Loss Unweighted: 3.3809 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.9791 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1767 | Regression_eta: 0.0142 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8296 | Classification_binary: 2.9791 | Regression_cos_phi: 0.5405 | Regression_energy: 1.9316 | Regression_eta: 0.6334 | Regression_pt: 2.0459 | Regression_sin_phi: 0.7840 | LR: 7.77e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.3483s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1197 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.3759 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3925.7 | uss_MB: 3440.4 | child_rss_MB: 3206.9 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:06:15,454] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:29,072] INFO : Step 16700/50000 rank0 | Train Loss: 10.7443 | Train Loss Unweighted: 3.6122 | Train Loss Components: Classification: 0.0223 | Classification_binary: 3.1313 | Regression_cos_phi: 0.0069 | Regression_energy: 0.2182 | Regression_eta: 0.0140 | Regression_pt: 0.2111 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8388 | Classification_binary: 3.1313 | Regression_cos_phi: 0.4451 | Regression_energy: 2.3850 | Regression_eta: 0.6225 | Regression_pt: 2.5082 | Regression_sin_phi: 0.8135 | LR: 7.74e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3490s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.1077 | max_backward: 0.0330 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3926.7 | uss_MB: 3439.4 | child_rss_MB: 3204.7 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:06:29,285] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:42,838] INFO : Step 16800/50000 rank0 | Train Loss: 9.9643 | Train Loss Unweighted: 3.4441 | Train Loss Components: Classification: 0.0215 | Classification_binary: 3.0469 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1718 | Regression_eta: 0.0189 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8107 | Classification_binary: 3.0469 | Regression_cos_phi: 0.5943 | Regression_energy: 1.8782 | Regression_eta: 0.8451 | Regression_pt: 1.9886 | Regression_sin_phi: 0.8006 | LR: 7.72e-04 | DataLoad Time: 0.0527s | Model Forward Time: 1.2436s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0035 | max_backward: 0.1412 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2787 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3926.7 | uss_MB: 3440.5 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:06:43,051] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:56,401] INFO : Step 16900/50000 rank0 | Train Loss: 9.4625 | Train Loss Unweighted: 3.3970 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.0477 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1479 | Regression_eta: 0.0210 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8314 | Classification_binary: 3.0477 | Regression_cos_phi: 0.5292 | Regression_energy: 1.6167 | Regression_eta: 0.9351 | Regression_pt: 1.6817 | Regression_sin_phi: 0.8206 | LR: 7.69e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.3045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0642 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3927.7 | uss_MB: 3443.0 | child_rss_MB: 3203.3 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:06:56,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:10,327] INFO : Step 17000/50000 rank0 | Train Loss: 9.6699 | Train Loss Unweighted: 3.4537 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0525 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1803 | Regression_eta: 0.0163 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8362 | Classification_binary: 3.0525 | Regression_cos_phi: 0.4481 | Regression_energy: 1.9716 | Regression_eta: 0.7289 | Regression_pt: 2.0052 | Regression_sin_phi: 0.6273 | LR: 7.67e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.3225s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0898 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3926.6 | uss_MB: 3440.4 | child_rss_MB: 3208.0 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:07:10,539] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:10,656] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 12:07:10,737] INFO : Running validation on rank0 +rank0 [2026-10-28 12:07:10,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:07:10,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:07:11,831] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 12:07:11,976] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:07:12,108] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:07:12,182] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:07:12,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:07:12,319] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:07:12,389] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:07:12,458] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:07:12,565] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:07:12,697] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:07:12,828] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:07:12,899] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:07:12,969] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:07:13,038] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:07:13,108] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:07:13,178] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:07:13,290] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:07:13,297] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:07:13,300] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:07:13,302] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:07:13,304] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:07:13,306] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:07:13,309] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:07:13,310] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:07:13,312] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:07:13,315] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:07:13,316] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:07:13,319] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:07:13,322] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:07:13,323] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:07:13,325] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:07:13,328] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:07:19,399] INFO : VALIDATION | Step=17000/50000 | Train Loss=9.6699 | Valid Loss=9.5219 | Train Loss Unweighted=3.4537 | Valid Loss Unweighted=3.4338 | Stale=0 | ETA=106.3m +rank0 [2026-10-28 12:07:19,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:07:19,464] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:07:19,468] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:07:19,621] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:07:21,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:07:21,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:07:22,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:07:22,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:07:23,067] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:07:23,070] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:07:23,125] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:07:23,128] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:07:23,287] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:07:24,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:07:25,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:07:25,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:07:26,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:07:26,439] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:07:26,442] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:07:26,495] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:07:26,498] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:07:26,587] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:07:27,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:07:28,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:07:29,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:07:29,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:07:29,648] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:07:29,655] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:07:29,915] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:07:30,320] INFO : Plotted number of elements +rank0 [2026-10-28 12:07:31,753] INFO : Plotted jets +rank0 [2026-10-28 12:07:32,832] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:32,865] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:07:33,095] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:07:33,489] INFO : Plotted number of elements +rank0 [2026-10-28 12:07:35,316] INFO : Plotted jets +rank0 [2026-10-28 12:07:36,422] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:36,454] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 12:07:36,662] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 12:07:37,027] INFO : Plotted number of elements +rank0 [2026-10-28 12:07:38,403] INFO : Plotted jets +rank0 [2026-10-28 12:07:39,450] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:53,199] INFO : Step 17100/50000 rank0 | Train Loss: 9.6612 | Train Loss Unweighted: 3.4378 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0353 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1802 | Regression_eta: 0.0137 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8367 | Classification_binary: 3.0353 | Regression_cos_phi: 0.4736 | Regression_energy: 1.9703 | Regression_eta: 0.6120 | Regression_pt: 2.0402 | Regression_sin_phi: 0.6931 | LR: 7.64e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1994s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.2821 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4132.1 | uss_MB: 3648.6 | child_rss_MB: 3205.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:07:53,425] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:06,564] INFO : Step 17200/50000 rank0 | Train Loss: 9.0897 | Train Loss Unweighted: 3.3886 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.0318 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1564 | Regression_eta: 0.0126 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8878 | Classification_binary: 3.0318 | Regression_cos_phi: 0.4772 | Regression_energy: 1.7094 | Regression_eta: 0.5614 | Regression_pt: 1.7843 | Regression_sin_phi: 0.6378 | LR: 7.62e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2642s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0033 | max_backward: 0.1201 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.2763 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4132.1 | uss_MB: 3648.6 | child_rss_MB: 3206.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:08:06,776] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:20,128] INFO : Step 17300/50000 rank0 | Train Loss: 9.2359 | Train Loss Unweighted: 3.4220 | Train Loss Components: Classification: 0.0197 | Classification_binary: 3.0338 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1743 | Regression_eta: 0.0120 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7414 | Classification_binary: 3.0338 | Regression_cos_phi: 0.4133 | Regression_energy: 1.9061 | Regression_eta: 0.5353 | Regression_pt: 2.0143 | Regression_sin_phi: 0.5916 | LR: 7.59e-04 | DataLoad Time: 0.0085s | Model Forward Time: 1.2761s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0361 | max_loss: 0.0035 | max_backward: 0.0330 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4132.1 | uss_MB: 3648.7 | child_rss_MB: 3205.9 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:08:20,343] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:33,798] INFO : Step 17400/50000 rank0 | Train Loss: 9.8186 | Train Loss Unweighted: 3.4440 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.0540 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1716 | Regression_eta: 0.0153 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8425 | Classification_binary: 3.0540 | Regression_cos_phi: 0.5882 | Regression_energy: 1.8764 | Regression_eta: 0.6808 | Regression_pt: 1.9333 | Regression_sin_phi: 0.8435 | LR: 7.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2385s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0849 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3996.6 | uss_MB: 3514.7 | child_rss_MB: 3228.1 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:08:34,012] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:47,683] INFO : Step 17500/50000 rank0 | Train Loss: 9.7547 | Train Loss Unweighted: 3.4522 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0522 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1746 | Regression_eta: 0.0123 | Regression_pt: 0.1735 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8577 | Classification_binary: 3.0522 | Regression_cos_phi: 0.5933 | Regression_energy: 1.9083 | Regression_eta: 0.5473 | Regression_pt: 2.0616 | Regression_sin_phi: 0.7342 | LR: 7.54e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2825s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0170 | max_loss: 0.0034 | max_backward: 0.1442 | max_optimizer_step: 0.1131 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3996.6 | uss_MB: 3516.0 | child_rss_MB: 3203.8 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:08:47,896] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:01,256] INFO : Step 17600/50000 rank0 | Train Loss: 9.1861 | Train Loss Unweighted: 3.3536 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.9685 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1721 | Regression_eta: 0.0096 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7876 | Classification_binary: 2.9685 | Regression_cos_phi: 0.4602 | Regression_energy: 1.8820 | Regression_eta: 0.4284 | Regression_pt: 2.0009 | Regression_sin_phi: 0.6586 | LR: 7.52e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3341s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0870 | max_loss: 0.0034 | max_backward: 0.0323 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.3799 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3997.6 | uss_MB: 3514.0 | child_rss_MB: 3204.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:01,471] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:14,911] INFO : Step 17700/50000 rank0 | Train Loss: 9.0973 | Train Loss Unweighted: 3.3805 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.0201 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1595 | Regression_eta: 0.0134 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7748 | Classification_binary: 3.0201 | Regression_cos_phi: 0.4690 | Regression_energy: 1.7438 | Regression_eta: 0.5975 | Regression_pt: 1.8111 | Regression_sin_phi: 0.6810 | LR: 7.49e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.1958s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1451 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3997.6 | uss_MB: 3514.1 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:09:15,137] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:28,207] INFO : Step 17800/50000 rank0 | Train Loss: 9.0643 | Train Loss Unweighted: 3.3253 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9493 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1697 | Regression_eta: 0.0123 | Regression_pt: 0.1598 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8047 | Classification_binary: 2.9493 | Regression_cos_phi: 0.4385 | Regression_energy: 1.8556 | Regression_eta: 0.5491 | Regression_pt: 1.8987 | Regression_sin_phi: 0.5685 | LR: 7.47e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.1909s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0034 | max_backward: 0.0324 | max_optimizer_step: 0.1612 | max_tensorboard_logging: 0.2722 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3514.3 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:09:28,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:41,520] INFO : Step 17900/50000 rank0 | Train Loss: 8.8853 | Train Loss Unweighted: 3.2572 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.9051 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1571 | Regression_eta: 0.0120 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7368 | Classification_binary: 2.9051 | Regression_cos_phi: 0.4958 | Regression_energy: 1.7178 | Regression_eta: 0.5362 | Regression_pt: 1.7582 | Regression_sin_phi: 0.7354 | LR: 7.44e-04 | DataLoad Time: 0.0390s | Model Forward Time: 1.2259s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0035 | max_backward: 0.1411 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.2636 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3514.4 | child_rss_MB: 3207.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:09:41,735] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:54,779] INFO : Step 18000/50000 rank0 | Train Loss: 10.1576 | Train Loss Unweighted: 3.4926 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.0708 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1847 | Regression_eta: 0.0169 | Regression_pt: 0.1822 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8180 | Classification_binary: 3.0708 | Regression_cos_phi: 0.4766 | Regression_energy: 2.0188 | Regression_eta: 0.7559 | Regression_pt: 2.1644 | Regression_sin_phi: 0.8530 | LR: 7.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1900s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1491 | max_tensorboard_logging: 0.2857 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3997.1 | uss_MB: 3513.0 | child_rss_MB: 3208.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:55,011] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:55,029] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 12:09:55,111] INFO : Running validation on rank0 +rank0 [2026-10-28 12:09:55,112] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:09:55,112] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:09:56,312] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 12:09:56,457] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:09:56,588] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:09:56,662] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:09:56,731] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:09:56,799] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:09:56,869] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:09:56,940] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:09:57,046] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:09:57,179] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:09:57,309] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:09:57,381] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:09:57,450] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:09:57,519] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:09:57,589] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:09:57,658] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:09:57,777] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:09:57,785] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:09:57,787] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:09:57,789] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:09:57,792] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:09:57,794] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:09:57,795] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:09:57,798] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:09:57,801] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:09:57,802] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:09:57,804] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:09:57,805] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:09:57,808] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:09:57,811] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:09:57,814] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:09:57,815] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:10:03,649] INFO : VALIDATION | Step=18000/50000 | Train Loss=10.1576 | Valid Loss=9.4119 | Train Loss Unweighted=3.4926 | Valid Loss Unweighted=3.4015 | Stale=0 | ETA=102.2m +rank0 [2026-10-28 12:10:03,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:03,714] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:10:03,718] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:10:03,875] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:05,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:05,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:06,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:07,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:07,273] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:10:07,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:07,332] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:10:07,335] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:07,498] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:08,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:09,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:10,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:10,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:10,763] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:10:10,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:10,821] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:10:10,825] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:10:10,989] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:12,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:13,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:13,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:14,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:14,287] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:10:14,294] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:10:14,551] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:10:14,956] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:16,397] INFO : Plotted jets +rank0 [2026-10-28 12:10:17,476] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:17,508] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:17,735] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:18,126] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:19,985] INFO : Plotted jets +rank0 [2026-10-28 12:10:21,088] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:21,120] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 12:10:21,326] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 12:10:21,691] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:23,072] INFO : Plotted jets +rank0 [2026-10-28 12:10:24,127] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:37,883] INFO : Step 18100/50000 rank0 | Train Loss: 8.9137 | Train Loss Unweighted: 3.2944 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.9349 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1615 | Regression_eta: 0.0101 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8298 | Classification_binary: 2.9349 | Regression_cos_phi: 0.4777 | Regression_energy: 1.7661 | Regression_eta: 0.4508 | Regression_pt: 1.8009 | Regression_sin_phi: 0.6535 | LR: 7.39e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2650s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4198.7 | uss_MB: 3715.8 | child_rss_MB: 3213.8 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:10:38,111] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:51,444] INFO : Step 18200/50000 rank0 | Train Loss: 9.4016 | Train Loss Unweighted: 3.3525 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9876 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1589 | Regression_eta: 0.0131 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8038 | Classification_binary: 2.9876 | Regression_cos_phi: 0.5212 | Regression_energy: 1.7372 | Regression_eta: 0.5857 | Regression_pt: 1.8252 | Regression_sin_phi: 0.9410 | LR: 7.36e-04 | DataLoad Time: 0.0532s | Model Forward Time: 1.3925s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.1095 | max_backward: 0.0343 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.3810 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4198.7 | uss_MB: 3714.7 | child_rss_MB: 3208.7 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:10:51,657] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:05,107] INFO : Step 18300/50000 rank0 | Train Loss: 9.9069 | Train Loss Unweighted: 3.4297 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.9981 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1951 | Regression_eta: 0.0100 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7769 | Classification_binary: 2.9981 | Regression_cos_phi: 0.4927 | Regression_energy: 2.1326 | Regression_eta: 0.4475 | Regression_pt: 2.2554 | Regression_sin_phi: 0.8037 | LR: 7.34e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0034 | max_backward: 0.0382 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.2818 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4198.7 | uss_MB: 3717.9 | child_rss_MB: 3213.4 | allocated_MB: 210.7 | reserved_MB: 2752.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:11:05,334] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:18,400] INFO : Step 18400/50000 rank0 | Train Loss: 9.6908 | Train Loss Unweighted: 3.3938 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0062 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1735 | Regression_eta: 0.0176 | Regression_pt: 0.1579 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8574 | Classification_binary: 3.0062 | Regression_cos_phi: 0.4922 | Regression_energy: 1.8972 | Regression_eta: 0.7842 | Regression_pt: 1.8757 | Regression_sin_phi: 0.7779 | LR: 7.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2230s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.3013 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4071.7 | uss_MB: 3587.5 | child_rss_MB: 3214.5 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:11:18,626] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:31,691] INFO : Step 18500/50000 rank0 | Train Loss: 9.0303 | Train Loss Unweighted: 3.3196 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.9558 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1590 | Regression_eta: 0.0082 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8504 | Classification_binary: 2.9558 | Regression_cos_phi: 0.5259 | Regression_energy: 1.7388 | Regression_eta: 0.3650 | Regression_pt: 1.8808 | Regression_sin_phi: 0.7136 | LR: 7.29e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1462 | max_tensorboard_logging: 0.2725 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4071.7 | uss_MB: 3588.6 | child_rss_MB: 3217.1 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:11:31,916] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:44,991] INFO : Step 18600/50000 rank0 | Train Loss: 9.0329 | Train Loss Unweighted: 3.3503 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.9951 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1556 | Regression_eta: 0.0089 | Regression_pt: 0.1534 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7863 | Classification_binary: 2.9951 | Regression_cos_phi: 0.4973 | Regression_energy: 1.7015 | Regression_eta: 0.3975 | Regression_pt: 1.8222 | Regression_sin_phi: 0.8332 | LR: 7.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2111s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0134 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4071.7 | uss_MB: 3588.7 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:11:45,222] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:58,371] INFO : Step 18700/50000 rank0 | Train Loss: 9.3181 | Train Loss Unweighted: 3.3776 | Train Loss Components: Classification: 0.0208 | Classification_binary: 3.0012 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1656 | Regression_eta: 0.0102 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 3.0012 | Regression_cos_phi: 0.5107 | Regression_energy: 1.8103 | Regression_eta: 0.4532 | Regression_pt: 1.9401 | Regression_sin_phi: 0.8177 | LR: 7.24e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2514s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.1231 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4071.7 | uss_MB: 3588.7 | child_rss_MB: 3212.0 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:11:58,583] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:11,835] INFO : Step 18800/50000 rank0 | Train Loss: 9.2646 | Train Loss Unweighted: 3.2165 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.8458 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1624 | Regression_eta: 0.0208 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7463 | Classification_binary: 2.8458 | Regression_cos_phi: 0.4257 | Regression_energy: 1.7749 | Regression_eta: 0.9298 | Regression_pt: 1.8250 | Regression_sin_phi: 0.7172 | LR: 7.21e-04 | DataLoad Time: 0.0787s | Model Forward Time: 1.3655s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0983 | max_backward: 0.0333 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4071.7 | uss_MB: 3587.5 | child_rss_MB: 3217.2 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:12:12,050] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:25,118] INFO : Step 18900/50000 rank0 | Train Loss: 9.5529 | Train Loss Unweighted: 3.3922 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9934 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1811 | Regression_eta: 0.0125 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7993 | Classification_binary: 2.9934 | Regression_cos_phi: 0.5456 | Regression_energy: 1.9798 | Regression_eta: 0.5564 | Regression_pt: 2.0015 | Regression_sin_phi: 0.6769 | LR: 7.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2966s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0517 | max_backward: 0.0334 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3757 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4071.7 | uss_MB: 3587.5 | child_rss_MB: 3209.5 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:12:25,331] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:38,204] INFO : Step 19000/50000 rank0 | Train Loss: 8.8477 | Train Loss Unweighted: 3.3325 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.9850 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1527 | Regression_eta: 0.0114 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8100 | Classification_binary: 2.9850 | Regression_cos_phi: 0.5107 | Regression_energy: 1.6694 | Regression_eta: 0.5066 | Regression_pt: 1.7533 | Regression_sin_phi: 0.6126 | LR: 7.16e-04 | DataLoad Time: 0.0643s | Model Forward Time: 1.2384s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.1244 | max_optimizer_step: 0.1145 | max_tensorboard_logging: 0.2775 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4071.3 | uss_MB: 3587.7 | child_rss_MB: 3215.2 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:12:38,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:38,435] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 12:12:38,516] INFO : Running validation on rank0 +rank0 [2026-10-28 12:12:38,516] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:12:38,516] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:12:39,717] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 12:12:39,862] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:12:39,994] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:12:40,068] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:12:40,137] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:12:40,205] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:12:40,275] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:12:40,346] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:12:40,455] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:12:40,586] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:12:40,717] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:12:40,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:12:40,859] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:12:40,928] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:12:40,997] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:12:41,068] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:12:41,180] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:12:41,187] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:12:41,190] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:12:41,191] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:12:41,194] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:12:41,196] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:12:41,198] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:12:41,200] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:12:41,203] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:12:41,204] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:12:41,207] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:12:41,209] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:12:41,210] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:12:41,213] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:12:41,216] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:12:41,217] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:12:47,127] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.8477 | Valid Loss=9.3071 | Train Loss Unweighted=3.3325 | Valid Loss Unweighted=3.3549 | Stale=0 | ETA=98.3m +rank0 [2026-10-28 12:12:47,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:47,194] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:12:47,197] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:47,357] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:48,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:49,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:50,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:50,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:50,745] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:12:50,748] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:50,804] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:12:50,807] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:50,897] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:52,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:52,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:53,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:54,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:54,239] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:12:54,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:54,296] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:12:54,299] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:12:54,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:55,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:56,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:56,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:57,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:57,527] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:12:57,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:57,796] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:58,201] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:59,641] INFO : Plotted jets +rank0 [2026-10-28 12:13:00,726] INFO : Plotted jet ratio +rank0 [2026-10-28 12:13:00,760] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:13:00,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:13:01,383] INFO : Plotted number of elements +rank0 [2026-10-28 12:13:03,220] INFO : Plotted jets +rank0 [2026-10-28 12:13:04,325] INFO : Plotted jet ratio +rank0 [2026-10-28 12:13:04,358] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 12:13:04,564] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 12:13:04,930] INFO : Plotted number of elements +rank0 [2026-10-28 12:13:06,305] INFO : Plotted jets +rank0 [2026-10-28 12:13:07,354] INFO : Plotted jet ratio +rank0 [2026-10-28 12:13:20,893] INFO : Step 19100/50000 rank0 | Train Loss: 9.3367 | Train Loss Unweighted: 3.3824 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0153 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1635 | Regression_eta: 0.0128 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8284 | Classification_binary: 3.0153 | Regression_cos_phi: 0.6239 | Regression_energy: 1.7871 | Regression_eta: 0.5692 | Regression_pt: 1.8038 | Regression_sin_phi: 0.7091 | LR: 7.13e-04 | DataLoad Time: 0.0765s | Model Forward Time: 1.2775s | Phase Times: max_pre_step_barrier: 0.0790 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4271.4 | uss_MB: 3787.2 | child_rss_MB: 3214.4 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:13:21,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:33,612] INFO : Step 19200/50000 rank0 | Train Loss: 8.8605 | Train Loss Unweighted: 3.3094 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.9431 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1633 | Regression_eta: 0.0104 | Regression_pt: 0.1599 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7335 | Classification_binary: 2.9431 | Regression_cos_phi: 0.4580 | Regression_energy: 1.7855 | Regression_eta: 0.4647 | Regression_pt: 1.9002 | Regression_sin_phi: 0.5754 | LR: 7.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2039s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0162 | max_loss: 0.0036 | max_backward: 0.0378 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4271.4 | uss_MB: 3789.3 | child_rss_MB: 3205.9 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:13:33,842] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:47,027] INFO : Step 19300/50000 rank0 | Train Loss: 9.1006 | Train Loss Unweighted: 3.2988 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.9370 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1595 | Regression_eta: 0.0122 | Regression_pt: 0.1539 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7631 | Classification_binary: 2.9370 | Regression_cos_phi: 0.5170 | Regression_energy: 1.7434 | Regression_eta: 0.5420 | Regression_pt: 1.8286 | Regression_sin_phi: 0.7695 | LR: 7.08e-04 | DataLoad Time: 0.0596s | Model Forward Time: 1.6940s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0444 | max_forward: 0.0175 | max_loss: 0.0034 | max_backward: 0.0387 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0802 | Memory: rss_MB: 4271.4 | uss_MB: 3789.4 | child_rss_MB: 3209.9 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:13:47,240] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:59,899] INFO : Step 19400/50000 rank0 | Train Loss: 9.6131 | Train Loss Unweighted: 3.3806 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.9920 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1710 | Regression_eta: 0.0129 | Regression_pt: 0.1641 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9003 | Classification_binary: 2.9920 | Regression_cos_phi: 0.5591 | Regression_energy: 1.8698 | Regression_eta: 0.5736 | Regression_pt: 1.9493 | Regression_sin_phi: 0.7689 | LR: 7.05e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3022s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0372 | max_forward: 0.0147 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3388 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4271.4 | uss_MB: 3787.3 | child_rss_MB: 3212.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:00,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:12,683] INFO : Step 19500/50000 rank0 | Train Loss: 8.8270 | Train Loss Unweighted: 3.2184 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8636 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1559 | Regression_eta: 0.0116 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7843 | Classification_binary: 2.8636 | Regression_cos_phi: 0.5091 | Regression_energy: 1.7041 | Regression_eta: 0.5176 | Regression_pt: 1.8040 | Regression_sin_phi: 0.6443 | LR: 7.02e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2183s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1424 | max_tensorboard_logging: 0.2952 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4148.1 | uss_MB: 3663.9 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:12,916] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:25,688] INFO : Step 19600/50000 rank0 | Train Loss: 9.1729 | Train Loss Unweighted: 3.2802 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9176 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1573 | Regression_eta: 0.0165 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7995 | Classification_binary: 2.9176 | Regression_cos_phi: 0.4890 | Regression_energy: 1.7199 | Regression_eta: 0.7362 | Regression_pt: 1.8131 | Regression_sin_phi: 0.6976 | LR: 7.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3488s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.3836 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4148.1 | uss_MB: 3664.0 | child_rss_MB: 3213.1 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:25,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:38,908] INFO : Step 19700/50000 rank0 | Train Loss: 9.5319 | Train Loss Unweighted: 3.3654 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.9858 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1680 | Regression_eta: 0.0183 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7715 | Classification_binary: 2.9858 | Regression_cos_phi: 0.4458 | Regression_energy: 1.8363 | Regression_eta: 0.8151 | Regression_pt: 1.8716 | Regression_sin_phi: 0.8059 | LR: 6.97e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3120s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0479 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4148.1 | uss_MB: 3664.1 | child_rss_MB: 3209.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:14:39,123] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:52,230] INFO : Step 19800/50000 rank0 | Train Loss: 9.9787 | Train Loss Unweighted: 3.3479 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.9566 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1642 | Regression_eta: 0.0248 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8351 | Classification_binary: 2.9566 | Regression_cos_phi: 0.5304 | Regression_energy: 1.7956 | Regression_eta: 1.1063 | Regression_pt: 1.9409 | Regression_sin_phi: 0.8137 | LR: 6.94e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0402 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3542 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4148.1 | uss_MB: 3664.1 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:14:52,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:05,152] INFO : Step 19900/50000 rank0 | Train Loss: 9.2461 | Train Loss Unweighted: 3.2684 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8965 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1624 | Regression_eta: 0.0160 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7676 | Classification_binary: 2.8965 | Regression_cos_phi: 0.4873 | Regression_energy: 1.7751 | Regression_eta: 0.7132 | Regression_pt: 1.8771 | Regression_sin_phi: 0.7292 | LR: 6.92e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2819s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.1317 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.2922 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4148.1 | uss_MB: 3664.2 | child_rss_MB: 3228.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:15:05,367] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:18,066] INFO : Step 20000/50000 rank0 | Train Loss: 10.4556 | Train Loss Unweighted: 3.5075 | Train Loss Components: Classification: 0.0216 | Classification_binary: 3.0704 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1958 | Regression_eta: 0.0202 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8136 | Classification_binary: 3.0704 | Regression_cos_phi: 0.5392 | Regression_energy: 2.1411 | Regression_eta: 0.9003 | Regression_pt: 2.1680 | Regression_sin_phi: 0.8230 | LR: 6.89e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3068s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0600 | max_forward: 0.0137 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.3531 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4148.1 | uss_MB: 3664.3 | child_rss_MB: 3203.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:15:18,278] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:18,314] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 12:15:18,396] INFO : Running validation on rank0 +rank0 [2026-10-28 12:15:18,397] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:15:18,397] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:15:19,503] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 12:15:19,647] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:15:19,781] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:15:19,854] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:15:19,926] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:15:19,994] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:15:20,064] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:15:20,134] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:15:20,242] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:15:20,376] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:15:20,507] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:15:20,580] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:15:20,649] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:15:20,718] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:15:20,789] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:15:20,859] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:15:20,975] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:15:20,982] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:15:20,984] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:15:20,988] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:15:20,990] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:15:20,991] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:15:20,994] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:15:20,995] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:15:20,998] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:15:21,001] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:15:21,003] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:15:21,006] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:15:21,009] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:15:21,010] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:15:21,013] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:15:21,016] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:15:27,088] INFO : VALIDATION | Step=20000/50000 | Train Loss=10.4556 | Valid Loss=9.2107 | Train Loss Unweighted=3.5075 | Valid Loss Unweighted=3.3124 | Stale=0 | ETA=94.3m +rank0 [2026-10-28 12:15:27,098] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:15:27,154] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:15:27,157] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:15:27,316] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:15:28,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:15:29,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:15:30,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:15:30,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:15:30,782] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:15:30,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:15:30,840] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:15:30,854] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:15:31,009] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:15:32,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:15:33,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:15:33,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:15:34,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:15:34,303] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:15:34,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:15:34,360] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:15:34,363] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:15:34,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:15:35,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:15:36,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:15:37,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:15:37,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:15:37,784] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:15:37,792] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:15:38,050] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:15:38,456] INFO : Plotted number of elements +rank0 [2026-10-28 12:15:39,901] INFO : Plotted jets +rank0 [2026-10-28 12:15:40,991] INFO : Plotted jet ratio +rank0 [2026-10-28 12:15:41,027] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:15:41,257] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:15:41,648] INFO : Plotted number of elements +rank0 [2026-10-28 12:15:43,503] INFO : Plotted jets +rank0 [2026-10-28 12:15:44,611] INFO : Plotted jet ratio +rank0 [2026-10-28 12:15:44,646] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 12:15:44,853] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 12:15:45,217] INFO : Plotted number of elements +rank0 [2026-10-28 12:15:46,602] INFO : Plotted jets +rank0 [2026-10-28 12:15:47,655] INFO : Plotted jet ratio +rank0 [2026-10-28 12:16:00,740] INFO : Step 20100/50000 rank0 | Train Loss: 9.7935 | Train Loss Unweighted: 3.3790 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.9950 | Regression_cos_phi: 0.0118 | Regression_energy: 0.1678 | Regression_eta: 0.0198 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7336 | Classification_binary: 2.9950 | Regression_cos_phi: 0.7653 | Regression_energy: 1.8348 | Regression_eta: 0.8833 | Regression_pt: 1.8715 | Regression_sin_phi: 0.7099 | LR: 6.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.3111 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4343.6 | uss_MB: 3860.9 | child_rss_MB: 3206.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:16:00,955] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:14,055] INFO : Step 20200/50000 rank0 | Train Loss: 9.1153 | Train Loss Unweighted: 3.2642 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8917 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1651 | Regression_eta: 0.0123 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7675 | Classification_binary: 2.8917 | Regression_cos_phi: 0.5092 | Regression_energy: 1.8052 | Regression_eta: 0.5470 | Regression_pt: 1.8947 | Regression_sin_phi: 0.6999 | LR: 6.84e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.3016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.1403 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4344.6 | uss_MB: 3861.0 | child_rss_MB: 3205.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:16:14,267] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:26,925] INFO : Step 20300/50000 rank0 | Train Loss: 9.5089 | Train Loss Unweighted: 3.4058 | Train Loss Components: Classification: 0.0230 | Classification_binary: 3.0174 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1721 | Regression_eta: 0.0124 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8664 | Classification_binary: 3.0174 | Regression_cos_phi: 0.5277 | Regression_energy: 1.8814 | Regression_eta: 0.5551 | Regression_pt: 1.9639 | Regression_sin_phi: 0.6969 | LR: 6.81e-04 | DataLoad Time: 0.0248s | Model Forward Time: 1.3971s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0801 | max_backward: 0.0331 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.4031 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4344.6 | uss_MB: 3860.9 | child_rss_MB: 3204.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:16:27,139] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:39,864] INFO : Step 20400/50000 rank0 | Train Loss: 9.5278 | Train Loss Unweighted: 3.3950 | Train Loss Components: Classification: 0.0199 | Classification_binary: 3.0173 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1664 | Regression_eta: 0.0121 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7495 | Classification_binary: 3.0173 | Regression_cos_phi: 0.5085 | Regression_energy: 1.8192 | Regression_eta: 0.5388 | Regression_pt: 1.9146 | Regression_sin_phi: 0.9799 | LR: 6.78e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3059s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0177 | max_loss: 0.0034 | max_backward: 0.0391 | max_optimizer_step: 0.1857 | max_tensorboard_logging: 0.2914 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4344.6 | uss_MB: 3863.1 | child_rss_MB: 3207.1 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:16:40,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:52,719] INFO : Step 20500/50000 rank0 | Train Loss: 9.2587 | Train Loss Unweighted: 3.3052 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9247 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1675 | Regression_eta: 0.0137 | Regression_pt: 0.1634 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8184 | Classification_binary: 2.9247 | Regression_cos_phi: 0.4553 | Regression_energy: 1.8317 | Regression_eta: 0.6095 | Regression_pt: 1.9420 | Regression_sin_phi: 0.6772 | LR: 6.76e-04 | DataLoad Time: 0.0040s | Model Forward Time: 1.3571s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0852 | max_backward: 0.0332 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4221.8 | uss_MB: 3735.5 | child_rss_MB: 3207.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:16:52,931] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:05,435] INFO : Step 20600/50000 rank0 | Train Loss: 8.3952 | Train Loss Unweighted: 3.2361 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.8944 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1512 | Regression_eta: 0.0076 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7867 | Classification_binary: 2.8944 | Regression_cos_phi: 0.3735 | Regression_energy: 1.6534 | Regression_eta: 0.3394 | Regression_pt: 1.7847 | Regression_sin_phi: 0.5632 | LR: 6.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0461 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4221.8 | uss_MB: 3736.8 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:17:05,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:18,438] INFO : Step 20700/50000 rank0 | Train Loss: 9.4802 | Train Loss Unweighted: 3.3733 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9964 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1660 | Regression_eta: 0.0142 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8445 | Classification_binary: 2.9964 | Regression_cos_phi: 0.4954 | Regression_energy: 1.8145 | Regression_eta: 0.6320 | Regression_pt: 1.8780 | Regression_sin_phi: 0.8194 | LR: 6.70e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2983s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.1410 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4221.8 | uss_MB: 3736.6 | child_rss_MB: 3207.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:17:18,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:31,061] INFO : Step 20800/50000 rank0 | Train Loss: 9.2801 | Train Loss Unweighted: 3.3489 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.9657 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1721 | Regression_eta: 0.0125 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7691 | Classification_binary: 2.9657 | Regression_cos_phi: 0.4627 | Regression_energy: 1.8812 | Regression_eta: 0.5562 | Regression_pt: 1.9449 | Regression_sin_phi: 0.7003 | LR: 6.67e-04 | DataLoad Time: 0.0575s | Model Forward Time: 1.2498s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0188 | max_loss: 0.0033 | max_backward: 0.0865 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.2864 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4221.8 | uss_MB: 3739.0 | child_rss_MB: 3230.2 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:17:31,275] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:43,668] INFO : Step 20900/50000 rank0 | Train Loss: 9.2390 | Train Loss Unweighted: 3.3702 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9963 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1663 | Regression_eta: 0.0112 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8181 | Classification_binary: 2.9963 | Regression_cos_phi: 0.4992 | Regression_energy: 1.8177 | Regression_eta: 0.5000 | Regression_pt: 1.8947 | Regression_sin_phi: 0.7131 | LR: 6.65e-04 | DataLoad Time: 0.0322s | Model Forward Time: 1.2085s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.0653 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.2939 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4220.8 | uss_MB: 3736.8 | child_rss_MB: 3229.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:17:43,880] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:56,301] INFO : Step 21000/50000 rank0 | Train Loss: 10.1144 | Train Loss Unweighted: 3.3631 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.9509 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1832 | Regression_eta: 0.0214 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7221 | Classification_binary: 2.9509 | Regression_cos_phi: 0.5213 | Regression_energy: 2.0025 | Regression_eta: 0.9554 | Regression_pt: 2.0268 | Regression_sin_phi: 0.9355 | LR: 6.62e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3371s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.1399 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3028 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 4220.8 | uss_MB: 3739.0 | child_rss_MB: 3228.2 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:17:56,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:56,536] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 12:17:56,616] INFO : Running validation on rank0 +rank0 [2026-10-28 12:17:56,617] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:17:56,617] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:17:57,808] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 12:17:57,954] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:17:58,086] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:17:58,161] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:17:58,230] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:17:58,300] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:17:58,370] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:17:58,440] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:17:58,548] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:17:58,671] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:17:58,792] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:17:58,863] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:17:58,933] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:17:59,002] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:17:59,071] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:17:59,141] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:17:59,257] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:17:59,265] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:17:59,268] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:17:59,271] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:17:59,272] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:17:59,274] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:17:59,277] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:17:59,279] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:17:59,281] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:17:59,284] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:17:59,286] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:17:59,287] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:17:59,290] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:17:59,293] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:17:59,294] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:17:59,296] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:18:05,229] INFO : VALIDATION | Step=21000/50000 | Train Loss=10.1144 | Valid Loss=9.2441 | Train Loss Unweighted=3.3631 | Valid Loss Unweighted=3.3157 | Stale=1 | ETA=90.5m +rank0 [2026-10-28 12:18:05,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:18:05,319] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:18:05,323] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:18:05,552] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:18:07,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:18:07,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:18:08,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:18:09,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:18:09,351] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:18:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:18:09,409] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:18:09,412] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:18:09,575] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:18:10,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:18:11,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:18:12,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:18:12,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:18:12,833] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:18:12,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:18:12,890] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:18:12,906] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:18:13,069] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:18:14,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:18:15,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:18:15,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:18:16,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:18:16,353] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:18:16,361] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:18:16,619] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:18:17,026] INFO : Plotted number of elements +rank0 [2026-10-28 12:18:18,471] INFO : Plotted jets +rank0 [2026-10-28 12:18:19,552] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:19,586] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:18:19,818] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:18:20,210] INFO : Plotted number of elements +rank0 [2026-10-28 12:18:22,100] INFO : Plotted jets +rank0 [2026-10-28 12:18:23,206] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:23,237] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 12:18:23,446] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 12:18:23,808] INFO : Plotted number of elements +rank0 [2026-10-28 12:18:25,191] INFO : Plotted jets +rank0 [2026-10-28 12:18:26,245] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:39,803] INFO : Step 21100/50000 rank0 | Train Loss: 9.5140 | Train Loss Unweighted: 3.3751 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.0026 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1626 | Regression_eta: 0.0135 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8000 | Classification_binary: 3.0026 | Regression_cos_phi: 0.6255 | Regression_energy: 1.7777 | Regression_eta: 0.6002 | Regression_pt: 1.8597 | Regression_sin_phi: 0.8482 | LR: 6.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0329 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4427.0 | uss_MB: 3945.3 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:18:40,033] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:41,296] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 12:18:41,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 12:18:51,233] INFO : Step 21200/50000 rank0 | Train Loss: 9.3283 | Train Loss Unweighted: 3.3616 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9888 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1660 | Regression_eta: 0.0146 | Regression_pt: 0.1560 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7970 | Classification_binary: 2.9888 | Regression_cos_phi: 0.4500 | Regression_energy: 1.8143 | Regression_eta: 0.6507 | Regression_pt: 1.8535 | Regression_sin_phi: 0.7739 | LR: 6.56e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3993s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0859 | max_backward: 0.0332 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3961 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4465.8 | uss_MB: 3983.2 | child_rss_MB: 3204.4 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:51,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:02,331] INFO : Step 21300/50000 rank0 | Train Loss: 8.6595 | Train Loss Unweighted: 3.2056 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.8543 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1554 | Regression_eta: 0.0103 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7761 | Classification_binary: 2.8543 | Regression_cos_phi: 0.4499 | Regression_energy: 1.6992 | Regression_eta: 0.4599 | Regression_pt: 1.7997 | Regression_sin_phi: 0.6204 | LR: 6.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3891s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0641 | max_backward: 0.0328 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.4129 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4465.8 | uss_MB: 3983.3 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:02,547] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:13,404] INFO : Step 21400/50000 rank0 | Train Loss: 8.4317 | Train Loss Unweighted: 3.1557 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.8291 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1430 | Regression_eta: 0.0098 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8022 | Classification_binary: 2.8291 | Regression_cos_phi: 0.4838 | Regression_energy: 1.5634 | Regression_eta: 0.4369 | Regression_pt: 1.6380 | Regression_sin_phi: 0.6784 | LR: 6.51e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.3229s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0220 | max_backward: 0.0333 | max_optimizer_step: 0.1131 | max_tensorboard_logging: 0.4030 | max_tensorboard_flush: 0.0131 | Memory: rss_MB: 4465.8 | uss_MB: 3983.3 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:13,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:24,484] INFO : Step 21500/50000 rank0 | Train Loss: 8.8374 | Train Loss Unweighted: 3.3401 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.9997 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1460 | Regression_eta: 0.0152 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8337 | Classification_binary: 2.9997 | Regression_cos_phi: 0.4296 | Regression_energy: 1.5961 | Regression_eta: 0.6764 | Regression_pt: 1.7137 | Regression_sin_phi: 0.5881 | LR: 6.48e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3635s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0661 | max_backward: 0.0331 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3980 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4291.4 | uss_MB: 3808.1 | child_rss_MB: 3199.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:19:24,697] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:35,619] INFO : Step 21600/50000 rank0 | Train Loss: 8.9794 | Train Loss Unweighted: 3.2363 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8504 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1734 | Regression_eta: 0.0092 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7697 | Classification_binary: 2.8504 | Regression_cos_phi: 0.4679 | Regression_energy: 1.8953 | Regression_eta: 0.4099 | Regression_pt: 2.0155 | Regression_sin_phi: 0.5707 | LR: 6.45e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3617s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0736 | max_backward: 0.0338 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3853 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4292.4 | uss_MB: 3807.6 | child_rss_MB: 3202.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:19:35,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:46,843] INFO : Step 21700/50000 rank0 | Train Loss: 8.7087 | Train Loss Unweighted: 3.2836 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.9215 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1626 | Regression_eta: 0.0109 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6924 | Classification_binary: 2.9215 | Regression_cos_phi: 0.3799 | Regression_energy: 1.7780 | Regression_eta: 0.4864 | Regression_pt: 1.8811 | Regression_sin_phi: 0.5694 | LR: 6.43e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3857s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0980 | max_backward: 0.0328 | max_optimizer_step: 0.1168 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4292.4 | uss_MB: 3808.8 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:19:47,056] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:58,096] INFO : Step 21800/50000 rank0 | Train Loss: 9.3929 | Train Loss Unweighted: 3.3557 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.9735 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1676 | Regression_eta: 0.0145 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8341 | Classification_binary: 2.9735 | Regression_cos_phi: 0.5217 | Regression_energy: 1.8322 | Regression_eta: 0.6463 | Regression_pt: 1.9389 | Regression_sin_phi: 0.6463 | LR: 6.40e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1202 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3921 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4292.4 | uss_MB: 3808.9 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:58,310] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:09,173] INFO : Step 21900/50000 rank0 | Train Loss: 9.7482 | Train Loss Unweighted: 3.3253 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.9334 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1709 | Regression_eta: 0.0224 | Regression_pt: 0.1621 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 2.9334 | Regression_cos_phi: 0.5531 | Regression_energy: 1.8686 | Regression_eta: 0.9979 | Regression_pt: 1.9255 | Regression_sin_phi: 0.6849 | LR: 6.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3250s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0697 | max_loss: 0.0031 | max_backward: 0.0331 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3948 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 4292.4 | uss_MB: 3809.0 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:20:09,385] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:20,234] INFO : Step 22000/50000 rank0 | Train Loss: 9.3696 | Train Loss Unweighted: 3.2712 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8855 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1700 | Regression_eta: 0.0132 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7633 | Classification_binary: 2.8855 | Regression_cos_phi: 0.5962 | Regression_energy: 1.8586 | Regression_eta: 0.5904 | Regression_pt: 1.9651 | Regression_sin_phi: 0.7104 | LR: 6.34e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.3109s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0573 | max_loss: 0.0029 | max_backward: 0.0328 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4292.4 | uss_MB: 3809.0 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:20:20,446] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:20,554] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 12:20:20,637] INFO : Running validation on rank0 +rank0 [2026-10-28 12:20:20,637] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:20:20,637] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:20:21,749] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 12:20:21,893] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:20:22,026] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:20:22,101] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:20:22,170] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:20:22,239] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:20:22,308] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:20:22,378] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:20:22,486] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:20:22,620] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:20:22,751] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:20:22,822] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:20:22,892] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:20:22,960] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:20:23,030] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:20:23,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:20:23,216] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:20:23,230] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:20:23,340] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:20:23,344] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:20:23,346] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:20:23,349] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:20:23,350] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:20:23,353] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:20:23,354] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:20:23,356] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:20:23,359] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:20:23,362] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:20:23,363] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:20:23,366] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:20:23,368] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:20:23,370] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:20:29,341] INFO : VALIDATION | Step=22000/50000 | Train Loss=9.3696 | Valid Loss=9.0184 | Train Loss Unweighted=3.2712 | Valid Loss Unweighted=3.2459 | Stale=0 | ETA=86.5m +rank0 [2026-10-28 12:20:29,350] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:20:29,408] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:20:29,412] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:20:29,576] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:20:31,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:20:31,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:20:32,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:20:32,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:20:32,994] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:20:32,998] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:20:33,056] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:20:33,060] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:20:33,224] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:20:34,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:20:35,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:20:35,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:20:36,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:20:36,494] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:20:36,498] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:20:36,553] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:20:36,556] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:20:36,725] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:20:38,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:20:38,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:20:39,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:20:39,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:20:39,848] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:20:39,855] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:20:40,115] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:20:40,522] INFO : Plotted number of elements +rank0 [2026-10-28 12:20:41,970] INFO : Plotted jets +rank0 [2026-10-28 12:20:43,051] INFO : Plotted jet ratio +rank0 [2026-10-28 12:20:43,084] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:20:43,314] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:20:43,710] INFO : Plotted number of elements +rank0 [2026-10-28 12:20:45,563] INFO : Plotted jets +rank0 [2026-10-28 12:20:46,671] INFO : Plotted jet ratio +rank0 [2026-10-28 12:20:46,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 12:20:46,911] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 12:20:47,275] INFO : Plotted number of elements +rank0 [2026-10-28 12:20:48,657] INFO : Plotted jets +rank0 [2026-10-28 12:20:49,710] INFO : Plotted jet ratio +rank0 [2026-10-28 12:21:00,922] INFO : Step 22100/50000 rank0 | Train Loss: 8.9134 | Train Loss Unweighted: 3.2596 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8998 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1598 | Regression_eta: 0.0095 | Regression_pt: 0.1556 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7214 | Classification_binary: 2.8998 | Regression_cos_phi: 0.4847 | Regression_energy: 1.7473 | Regression_eta: 0.4252 | Regression_pt: 1.8485 | Regression_sin_phi: 0.7866 | LR: 6.32e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0676 | max_backward: 0.0339 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.4074 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4498.8 | uss_MB: 4016.8 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:21:01,136] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:12,119] INFO : Step 22200/50000 rank0 | Train Loss: 9.3864 | Train Loss Unweighted: 3.3148 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.9348 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1678 | Regression_eta: 0.0122 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8401 | Classification_binary: 2.9348 | Regression_cos_phi: 0.4851 | Regression_energy: 1.8345 | Regression_eta: 0.5452 | Regression_pt: 1.9186 | Regression_sin_phi: 0.8281 | LR: 6.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3763s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0864 | max_loss: 0.0030 | max_backward: 0.0341 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4498.8 | uss_MB: 4016.8 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:21:12,330] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:23,179] INFO : Step 22300/50000 rank0 | Train Loss: 9.4579 | Train Loss Unweighted: 3.3217 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.9623 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1590 | Regression_eta: 0.0161 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8285 | Classification_binary: 2.9623 | Regression_cos_phi: 0.5194 | Regression_energy: 1.7380 | Regression_eta: 0.7175 | Regression_pt: 1.7115 | Regression_sin_phi: 0.9807 | LR: 6.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4367s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0809 | max_backward: 0.0330 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.4043 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4498.8 | uss_MB: 4016.8 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:21:23,391] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:34,340] INFO : Step 22400/50000 rank0 | Train Loss: 9.1994 | Train Loss Unweighted: 3.2721 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.9160 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1549 | Regression_eta: 0.0158 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9057 | Classification_binary: 2.9160 | Regression_cos_phi: 0.5012 | Regression_energy: 1.6931 | Regression_eta: 0.7027 | Regression_pt: 1.7327 | Regression_sin_phi: 0.7480 | LR: 6.23e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3340s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0942 | max_forward: 0.0170 | max_loss: 0.0033 | max_backward: 0.0386 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3262 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4498.8 | uss_MB: 4019.0 | child_rss_MB: 3203.3 | allocated_MB: 210.7 | reserved_MB: 2838.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:21:34,551] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:45,550] INFO : Step 22500/50000 rank0 | Train Loss: 8.9462 | Train Loss Unweighted: 3.2222 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8741 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1500 | Regression_eta: 0.0161 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7491 | Classification_binary: 2.8741 | Regression_cos_phi: 0.5615 | Regression_energy: 1.6395 | Regression_eta: 0.7162 | Regression_pt: 1.7405 | Regression_sin_phi: 0.6652 | LR: 6.20e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3929s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0469 | max_forward: 0.0155 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 4366.8 | uss_MB: 3883.9 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:21:45,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:56,558] INFO : Step 22600/50000 rank0 | Train Loss: 8.8771 | Train Loss Unweighted: 3.2326 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.8879 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1506 | Regression_eta: 0.0140 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7522 | Classification_binary: 2.8879 | Regression_cos_phi: 0.5052 | Regression_energy: 1.6469 | Regression_eta: 0.6252 | Regression_pt: 1.7166 | Regression_sin_phi: 0.7432 | LR: 6.18e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3467s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1032 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3421 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4366.8 | uss_MB: 3884.8 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:21:56,770] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:07,503] INFO : Step 22700/50000 rank0 | Train Loss: 9.2684 | Train Loss Unweighted: 3.2245 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.8495 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1645 | Regression_eta: 0.0120 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7862 | Classification_binary: 2.8495 | Regression_cos_phi: 0.5305 | Regression_energy: 1.7983 | Regression_eta: 0.5372 | Regression_pt: 1.9054 | Regression_sin_phi: 0.8613 | LR: 6.15e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3713s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0793 | max_backward: 0.0329 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3957 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4366.8 | uss_MB: 3884.8 | child_rss_MB: 3201.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:22:07,715] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:18,678] INFO : Step 22800/50000 rank0 | Train Loss: 9.0219 | Train Loss Unweighted: 3.3442 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.9783 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1626 | Regression_eta: 0.0106 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7243 | Classification_binary: 2.9783 | Regression_cos_phi: 0.4605 | Regression_energy: 1.7777 | Regression_eta: 0.4736 | Regression_pt: 1.8867 | Regression_sin_phi: 0.7208 | LR: 6.12e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1198 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.3919 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 4366.8 | uss_MB: 3885.0 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:22:18,895] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:22:29,876] INFO : Step 22900/50000 rank0 | Train Loss: 9.9238 | Train Loss Unweighted: 3.3371 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9352 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1763 | Regression_eta: 0.0165 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8435 | Classification_binary: 2.9352 | Regression_cos_phi: 0.7334 | Regression_energy: 1.9276 | Regression_eta: 0.7340 | Regression_pt: 1.9886 | Regression_sin_phi: 0.7615 | LR: 6.09e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3849s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0929 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3458 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4366.8 | uss_MB: 3885.1 | child_rss_MB: 3199.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:22:30,089] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:40,890] INFO : Step 23000/50000 rank0 | Train Loss: 8.3613 | Train Loss Unweighted: 3.1511 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.8175 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1479 | Regression_eta: 0.0095 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7415 | Classification_binary: 2.8175 | Regression_cos_phi: 0.3965 | Regression_energy: 1.6172 | Regression_eta: 0.4238 | Regression_pt: 1.7043 | Regression_sin_phi: 0.6606 | LR: 6.06e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0458 | max_loss: 0.0033 | max_backward: 0.0382 | max_optimizer_step: 0.1154 | max_tensorboard_logging: 0.3887 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4366.8 | uss_MB: 3887.6 | child_rss_MB: 3204.5 | allocated_MB: 210.7 | reserved_MB: 2838.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:22:41,101] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:41,182] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 12:22:41,264] INFO : Running validation on rank0 +rank0 [2026-10-28 12:22:41,264] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:22:41,264] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:22:42,369] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 12:22:42,514] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:22:42,647] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:22:42,722] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:22:42,792] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:22:42,859] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:22:42,929] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:22:42,999] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:22:43,108] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:22:43,242] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:22:43,373] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:22:43,444] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:22:43,515] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:22:43,584] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:22:43,654] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:22:43,724] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:22:43,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:22:43,851] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:22:43,853] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:22:43,856] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:22:43,857] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:22:43,860] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:22:43,863] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:22:43,864] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:22:43,866] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:22:43,870] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:22:43,871] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:22:43,873] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:22:43,876] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:22:43,877] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:22:43,879] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:22:43,882] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:22:49,956] INFO : VALIDATION | Step=23000/50000 | Train Loss=8.3613 | Valid Loss=8.9301 | Train Loss Unweighted=3.1511 | Valid Loss Unweighted=3.2115 | Stale=0 | ETA=82.5m +rank0 [2026-10-28 12:22:49,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:22:50,025] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:22:50,028] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:22:50,189] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:22:51,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:22:52,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:22:52,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:22:53,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:22:53,691] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:22:53,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:22:53,748] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:22:53,751] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:22:53,863] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:22:55,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:22:55,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:22:56,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:22:57,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:22:57,216] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:22:57,220] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:22:57,273] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:22:57,276] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:22:57,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:22:58,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:22:59,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:22:59,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:23:00,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:23:00,569] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:23:00,576] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:23:00,838] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:23:01,246] INFO : Plotted number of elements +rank0 [2026-10-28 12:23:02,686] INFO : Plotted jets +rank0 [2026-10-28 12:23:03,774] INFO : Plotted jet ratio +rank0 [2026-10-28 12:23:03,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:23:04,036] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:23:04,427] INFO : Plotted number of elements +rank0 [2026-10-28 12:23:06,274] INFO : Plotted jets +rank0 [2026-10-28 12:23:07,374] INFO : Plotted jet ratio +rank0 [2026-10-28 12:23:07,406] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 12:23:07,612] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 12:23:07,976] INFO : Plotted number of elements +rank0 [2026-10-28 12:23:09,356] INFO : Plotted jets +rank0 [2026-10-28 12:23:10,414] INFO : Plotted jet ratio +rank0 [2026-10-28 12:23:21,446] INFO : Step 23100/50000 rank0 | Train Loss: 9.6468 | Train Loss Unweighted: 3.3150 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.9471 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1582 | Regression_eta: 0.0201 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7170 | Classification_binary: 2.9471 | Regression_cos_phi: 0.7279 | Regression_energy: 1.7298 | Regression_eta: 0.8972 | Regression_pt: 1.7866 | Regression_sin_phi: 0.8412 | LR: 6.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3585s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0497 | max_backward: 0.0329 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4571.1 | uss_MB: 4089.5 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:21,656] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:32,348] INFO : Step 23200/50000 rank0 | Train Loss: 8.9469 | Train Loss Unweighted: 3.3141 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.9570 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1575 | Regression_eta: 0.0103 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8189 | Classification_binary: 2.9570 | Regression_cos_phi: 0.4934 | Regression_energy: 1.7223 | Regression_eta: 0.4572 | Regression_pt: 1.8153 | Regression_sin_phi: 0.6827 | LR: 6.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0032 | max_backward: 0.0573 | max_optimizer_step: 0.1190 | max_tensorboard_logging: 0.2867 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4571.1 | uss_MB: 4089.6 | child_rss_MB: 3226.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:23:32,562] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:43,068] INFO : Step 23300/50000 rank0 | Train Loss: 8.5894 | Train Loss Unweighted: 3.1996 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.8411 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1622 | Regression_eta: 0.0069 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7537 | Classification_binary: 2.8411 | Regression_cos_phi: 0.4005 | Regression_energy: 1.7730 | Regression_eta: 0.3090 | Regression_pt: 1.8579 | Regression_sin_phi: 0.6542 | LR: 5.98e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2683s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0491 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3069 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4571.1 | uss_MB: 4089.7 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:43,283] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:54,049] INFO : Step 23400/50000 rank0 | Train Loss: 8.3101 | Train Loss Unweighted: 3.2046 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.8788 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1428 | Regression_eta: 0.0094 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7583 | Classification_binary: 2.8788 | Regression_cos_phi: 0.4622 | Regression_energy: 1.5607 | Regression_eta: 0.4187 | Regression_pt: 1.6689 | Regression_sin_phi: 0.5624 | LR: 5.95e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3291s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0163 | max_loss: 0.0382 | max_backward: 0.0381 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4571.1 | uss_MB: 4091.7 | child_rss_MB: 3201.5 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:23:54,261] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:05,280] INFO : Step 23500/50000 rank0 | Train Loss: 8.7530 | Train Loss Unweighted: 3.2836 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.9226 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1603 | Regression_eta: 0.0087 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8124 | Classification_binary: 2.9226 | Regression_cos_phi: 0.3815 | Regression_energy: 1.7525 | Regression_eta: 0.3876 | Regression_pt: 1.8774 | Regression_sin_phi: 0.6190 | LR: 5.92e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4104s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1050 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1194 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4571.1 | uss_MB: 4089.7 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:24:05,494] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:16,417] INFO : Step 23600/50000 rank0 | Train Loss: 8.7641 | Train Loss Unweighted: 3.1443 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7951 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1559 | Regression_eta: 0.0155 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7648 | Classification_binary: 2.7951 | Regression_cos_phi: 0.4720 | Regression_energy: 1.7046 | Regression_eta: 0.6912 | Regression_pt: 1.7055 | Regression_sin_phi: 0.6310 | LR: 5.90e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0455 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0344 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.3225 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4440.4 | uss_MB: 3958.0 | child_rss_MB: 3202.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:24:16,632] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:27,494] INFO : Step 23700/50000 rank0 | Train Loss: 8.9924 | Train Loss Unweighted: 3.2108 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8319 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1675 | Regression_eta: 0.0117 | Regression_pt: 0.1661 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7644 | Classification_binary: 2.8319 | Regression_cos_phi: 0.4139 | Regression_energy: 1.8309 | Regression_eta: 0.5222 | Regression_pt: 1.9732 | Regression_sin_phi: 0.6558 | LR: 5.87e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3146s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0903 | max_forward: 0.0180 | max_loss: 0.0032 | max_backward: 0.0384 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3066 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4440.4 | uss_MB: 3960.7 | child_rss_MB: 3205.0 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:24:27,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:38,560] INFO : Step 23800/50000 rank0 | Train Loss: 9.5199 | Train Loss Unweighted: 3.2353 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8413 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1766 | Regression_eta: 0.0177 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7676 | Classification_binary: 2.8413 | Regression_cos_phi: 0.4759 | Regression_energy: 1.9310 | Regression_eta: 0.7908 | Regression_pt: 1.9460 | Regression_sin_phi: 0.7672 | LR: 5.84e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1146 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1175 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4440.4 | uss_MB: 3958.6 | child_rss_MB: 3208.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:24:38,772] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:49,687] INFO : Step 23900/50000 rank0 | Train Loss: 9.0106 | Train Loss Unweighted: 3.2836 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.9127 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1643 | Regression_eta: 0.0105 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7776 | Classification_binary: 2.9127 | Regression_cos_phi: 0.4456 | Regression_energy: 1.7965 | Regression_eta: 0.4703 | Regression_pt: 1.9147 | Regression_sin_phi: 0.6932 | LR: 5.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3131s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1065 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1179 | max_tensorboard_logging: 0.3110 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 4440.4 | uss_MB: 3958.7 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:24:49,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:00,449] INFO : Step 24000/50000 rank0 | Train Loss: 9.4634 | Train Loss Unweighted: 3.1679 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.7859 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1666 | Regression_eta: 0.0152 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9082 | Classification_binary: 2.7859 | Regression_cos_phi: 0.5432 | Regression_energy: 1.8208 | Regression_eta: 0.6787 | Regression_pt: 1.8872 | Regression_sin_phi: 0.8393 | LR: 5.78e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2057s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0720 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.2704 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 4440.4 | uss_MB: 3958.4 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:25:00,662] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:00,681] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 12:25:00,762] INFO : Running validation on rank0 +rank0 [2026-10-28 12:25:00,762] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:25:00,762] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:25:01,957] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 12:25:02,103] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:25:02,235] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:25:02,309] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:25:02,378] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:25:02,446] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:25:02,516] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:25:02,587] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:25:02,696] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:25:02,829] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:25:02,961] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:25:03,033] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:25:03,102] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:25:03,171] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:25:03,241] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:25:03,311] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:25:03,414] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:25:03,421] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:25:03,424] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:25:03,425] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:25:03,428] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:25:03,430] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:25:03,431] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:25:03,434] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:25:03,437] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:25:03,440] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:25:03,441] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:25:03,444] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:25:03,446] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:25:03,447] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:25:03,450] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:25:03,452] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:25:09,320] INFO : VALIDATION | Step=24000/50000 | Train Loss=9.4634 | Valid Loss=9.0209 | Train Loss Unweighted=3.1679 | Valid Loss Unweighted=3.2470 | Stale=1 | ETA=78.6m +rank0 [2026-10-28 12:25:09,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:25:09,388] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:25:09,391] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:25:09,458] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:25:10,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:25:11,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:25:12,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:25:12,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:12,896] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:25:12,899] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:25:12,954] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:25:12,958] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:13,120] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:25:14,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:25:15,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:25:15,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:25:16,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:16,389] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:25:16,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:25:16,445] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:25:16,465] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:25:16,628] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:25:18,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:25:18,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:25:19,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:25:19,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:19,949] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:25:19,956] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:25:20,218] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:25:20,628] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:22,069] INFO : Plotted jets +rank0 [2026-10-28 12:25:23,156] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:23,188] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:23,417] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:23,811] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:25,706] INFO : Plotted jets +rank0 [2026-10-28 12:25:26,811] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:26,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 12:25:27,049] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 12:25:27,412] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:28,794] INFO : Plotted jets +rank0 [2026-10-28 12:25:29,842] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:40,995] INFO : Step 24100/50000 rank0 | Train Loss: 9.4241 | Train Loss Unweighted: 3.3252 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.9290 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1753 | Regression_eta: 0.0100 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7773 | Classification_binary: 2.9290 | Regression_cos_phi: 0.5549 | Regression_energy: 1.9164 | Regression_eta: 0.4454 | Regression_pt: 2.0666 | Regression_sin_phi: 0.7345 | LR: 5.75e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3890s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0927 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1190 | max_tensorboard_logging: 0.3819 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4632.0 | uss_MB: 4149.9 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:25:41,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:51,973] INFO : Step 24200/50000 rank0 | Train Loss: 9.3717 | Train Loss Unweighted: 3.2718 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8826 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1732 | Regression_eta: 0.0143 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7474 | Classification_binary: 2.8826 | Regression_cos_phi: 0.5631 | Regression_energy: 1.8938 | Regression_eta: 0.6372 | Regression_pt: 1.9722 | Regression_sin_phi: 0.6754 | LR: 5.73e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3970s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0862 | max_backward: 0.0332 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.3760 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4632.0 | uss_MB: 4149.9 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:25:52,186] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:03,443] INFO : Step 24300/50000 rank0 | Train Loss: 9.0756 | Train Loss Unweighted: 3.2414 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.8783 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1605 | Regression_eta: 0.0124 | Regression_pt: 0.1544 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7296 | Classification_binary: 2.8783 | Regression_cos_phi: 0.4774 | Regression_energy: 1.7549 | Regression_eta: 0.5528 | Regression_pt: 1.8351 | Regression_sin_phi: 0.8475 | LR: 5.70e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3141s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.2288 | max_tensorboard_logging: 0.2810 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4632.0 | uss_MB: 4149.9 | child_rss_MB: 3201.9 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:26:03,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:14,303] INFO : Step 24400/50000 rank0 | Train Loss: 9.3651 | Train Loss Unweighted: 3.2259 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8516 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1620 | Regression_eta: 0.0202 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7842 | Classification_binary: 2.8516 | Regression_cos_phi: 0.4701 | Regression_energy: 1.7714 | Regression_eta: 0.8994 | Regression_pt: 1.8572 | Regression_sin_phi: 0.7313 | LR: 5.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3048s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0713 | max_forward: 0.0152 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3121 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4630.9 | uss_MB: 4149.7 | child_rss_MB: 3203.9 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:26:14,519] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:25,431] INFO : Step 24500/50000 rank0 | Train Loss: 9.2805 | Train Loss Unweighted: 3.1733 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.8069 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1592 | Regression_eta: 0.0139 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8133 | Classification_binary: 2.8069 | Regression_cos_phi: 0.6099 | Regression_energy: 1.7403 | Regression_eta: 0.6220 | Regression_pt: 1.8183 | Regression_sin_phi: 0.8698 | LR: 5.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3311s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0868 | max_forward: 0.0152 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3087 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4630.9 | uss_MB: 4149.8 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:26:25,643] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:36,495] INFO : Step 24600/50000 rank0 | Train Loss: 8.8440 | Train Loss Unweighted: 3.2523 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.9056 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1509 | Regression_eta: 0.0112 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7726 | Classification_binary: 2.9056 | Regression_cos_phi: 0.4615 | Regression_energy: 1.6494 | Regression_eta: 0.5013 | Regression_pt: 1.7657 | Regression_sin_phi: 0.7878 | LR: 5.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0468 | max_loss: 0.0031 | max_backward: 0.0385 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4510.3 | uss_MB: 4033.3 | child_rss_MB: 3202.4 | allocated_MB: 210.7 | reserved_MB: 2756.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:26:36,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:47,588] INFO : Step 24700/50000 rank0 | Train Loss: 9.5412 | Train Loss Unweighted: 3.1693 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.7823 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1690 | Regression_eta: 0.0150 | Regression_pt: 0.1619 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8576 | Classification_binary: 2.7823 | Regression_cos_phi: 0.6001 | Regression_energy: 1.8472 | Regression_eta: 0.6712 | Regression_pt: 1.9241 | Regression_sin_phi: 0.8587 | LR: 5.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3453s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0597 | max_loss: 0.0033 | max_backward: 0.0392 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4510.3 | uss_MB: 4033.7 | child_rss_MB: 3198.3 | allocated_MB: 210.7 | reserved_MB: 2756.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:26:47,805] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:58,677] INFO : Step 24800/50000 rank0 | Train Loss: 9.1292 | Train Loss Unweighted: 3.2207 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8518 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1627 | Regression_eta: 0.0092 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8064 | Classification_binary: 2.8518 | Regression_cos_phi: 0.4619 | Regression_energy: 1.7789 | Regression_eta: 0.4100 | Regression_pt: 1.8852 | Regression_sin_phi: 0.9350 | LR: 5.56e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2420s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0403 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1189 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4512.3 | uss_MB: 4032.4 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:26:58,891] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:09,697] INFO : Step 24900/50000 rank0 | Train Loss: 9.0073 | Train Loss Unweighted: 3.2081 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.8454 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1587 | Regression_eta: 0.0147 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6775 | Classification_binary: 2.8454 | Regression_cos_phi: 0.4963 | Regression_energy: 1.7350 | Regression_eta: 0.6562 | Regression_pt: 1.8502 | Regression_sin_phi: 0.7466 | LR: 5.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3435s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0237 | max_backward: 0.0329 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3994 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4512.3 | uss_MB: 4032.5 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:27:09,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:20,889] INFO : Step 25000/50000 rank0 | Train Loss: 9.1526 | Train Loss Unweighted: 3.2579 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8844 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1673 | Regression_eta: 0.0126 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7229 | Classification_binary: 2.8844 | Regression_cos_phi: 0.5866 | Regression_energy: 1.8289 | Regression_eta: 0.5622 | Regression_pt: 1.8784 | Regression_sin_phi: 0.6892 | LR: 5.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3232s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0419 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.3845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4513.3 | uss_MB: 4032.6 | child_rss_MB: 3198.8 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:21,100] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:21,174] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 12:27:21,255] INFO : Running validation on rank0 +rank0 [2026-10-28 12:27:21,256] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:27:21,256] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:27:22,345] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 12:27:22,491] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:27:22,624] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:27:22,698] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:27:22,768] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:27:22,836] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:27:22,905] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:27:22,975] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:27:23,083] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:27:23,207] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:27:23,329] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:27:23,402] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:27:23,472] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:27:23,541] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:27:23,610] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:27:23,680] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:27:23,798] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:27:23,805] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:27:23,808] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:27:23,811] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:27:23,812] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:27:23,814] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:27:23,817] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:27:23,818] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:27:23,821] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:27:23,824] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:27:23,826] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:27:23,827] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:27:23,830] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:27:23,831] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:27:23,833] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:27:23,836] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:27:29,665] INFO : VALIDATION | Step=25000/50000 | Train Loss=9.1526 | Valid Loss=8.9001 | Train Loss Unweighted=3.2579 | Valid Loss Unweighted=3.2063 | Stale=0 | ETA=74.9m +rank0 [2026-10-28 12:27:29,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:27:29,730] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:27:29,733] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:27:29,892] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:27:31,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:27:32,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:27:32,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:27:33,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:27:33,637] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:27:33,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:27:33,695] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:27:33,709] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:33,880] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:27:35,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:27:35,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:27:36,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:27:37,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:27:37,126] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:27:37,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:27:37,182] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:27:37,194] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:27:37,355] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:27:38,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:27:39,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:27:39,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:27:40,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:27:40,674] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:27:40,681] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:27:40,944] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:27:41,352] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:42,795] INFO : Plotted jets +rank0 [2026-10-28 12:27:43,880] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:43,914] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:44,144] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:44,540] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:46,463] INFO : Plotted jets +rank0 [2026-10-28 12:27:47,572] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:47,617] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:47,829] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:48,192] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:49,579] INFO : Plotted jets +rank0 [2026-10-28 12:27:50,637] INFO : Plotted jet ratio +rank0 [2026-10-28 12:28:01,696] INFO : Step 25100/50000 rank0 | Train Loss: 8.2584 | Train Loss Unweighted: 3.0014 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.6806 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1374 | Regression_eta: 0.0114 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8410 | Classification_binary: 2.6806 | Regression_cos_phi: 0.4856 | Regression_energy: 1.5021 | Regression_eta: 0.5076 | Regression_pt: 1.6108 | Regression_sin_phi: 0.6308 | LR: 5.47e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0789 | max_backward: 0.0332 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3825 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4703.2 | uss_MB: 4226.0 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:01,907] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:12,894] INFO : Step 25200/50000 rank0 | Train Loss: 8.6830 | Train Loss Unweighted: 3.2349 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8900 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1529 | Regression_eta: 0.0110 | Regression_pt: 0.1471 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7216 | Classification_binary: 2.8900 | Regression_cos_phi: 0.5159 | Regression_energy: 1.6715 | Regression_eta: 0.4886 | Regression_pt: 1.7477 | Regression_sin_phi: 0.6477 | LR: 5.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3405s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0367 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3770 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4703.2 | uss_MB: 4228.2 | child_rss_MB: 3196.5 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:28:13,108] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:23,966] INFO : Step 25300/50000 rank0 | Train Loss: 9.2545 | Train Loss Unweighted: 3.1495 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7586 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1750 | Regression_eta: 0.0125 | Regression_pt: 0.1677 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7309 | Classification_binary: 2.7586 | Regression_cos_phi: 0.5234 | Regression_energy: 1.9128 | Regression_eta: 0.5563 | Regression_pt: 1.9929 | Regression_sin_phi: 0.7796 | LR: 5.42e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3667s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0997 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3734 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4703.2 | uss_MB: 4226.1 | child_rss_MB: 3193.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:28:24,182] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:34,753] INFO : Step 25400/50000 rank0 | Train Loss: 8.6752 | Train Loss Unweighted: 3.2080 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.8626 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1555 | Regression_eta: 0.0106 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7418 | Classification_binary: 2.8626 | Regression_cos_phi: 0.4412 | Regression_energy: 1.7003 | Regression_eta: 0.4726 | Regression_pt: 1.7237 | Regression_sin_phi: 0.7330 | LR: 5.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2264s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0785 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.2724 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4703.2 | uss_MB: 4226.0 | child_rss_MB: 3215.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:34,967] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:45,775] INFO : Step 25500/50000 rank0 | Train Loss: 8.9248 | Train Loss Unweighted: 3.1332 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.7742 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1571 | Regression_eta: 0.0143 | Regression_pt: 0.1513 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7880 | Classification_binary: 2.7742 | Regression_cos_phi: 0.5333 | Regression_energy: 1.7173 | Regression_eta: 0.6357 | Regression_pt: 1.7977 | Regression_sin_phi: 0.6786 | LR: 5.36e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2894s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4703.2 | uss_MB: 4226.2 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:46,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:56,923] INFO : Step 25600/50000 rank0 | Train Loss: 8.1805 | Train Loss Unweighted: 3.0910 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7747 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1393 | Regression_eta: 0.0112 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7115 | Classification_binary: 2.7747 | Regression_cos_phi: 0.5315 | Regression_energy: 1.5230 | Regression_eta: 0.4974 | Regression_pt: 1.5779 | Regression_sin_phi: 0.5646 | LR: 5.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2689s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.2763 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 4583.5 | uss_MB: 4106.1 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:28:57,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:08,075] INFO : Step 25700/50000 rank0 | Train Loss: 9.5677 | Train Loss Unweighted: 3.2010 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.8236 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1667 | Regression_eta: 0.0250 | Regression_pt: 0.1498 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7357 | Classification_binary: 2.8236 | Regression_cos_phi: 0.5658 | Regression_energy: 1.8227 | Regression_eta: 1.1150 | Regression_pt: 1.7794 | Regression_sin_phi: 0.7255 | LR: 5.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2719s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0340 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.2714 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4583.5 | uss_MB: 4106.2 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:29:08,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:19,223] INFO : Step 25800/50000 rank0 | Train Loss: 8.6022 | Train Loss Unweighted: 3.1731 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8376 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1474 | Regression_eta: 0.0119 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7261 | Classification_binary: 2.8376 | Regression_cos_phi: 0.4433 | Regression_energy: 1.6111 | Regression_eta: 0.5329 | Regression_pt: 1.6867 | Regression_sin_phi: 0.7644 | LR: 5.27e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2550s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0344 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.2726 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4584.5 | uss_MB: 4106.8 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:29:19,451] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:30,128] INFO : Step 25900/50000 rank0 | Train Loss: 9.0806 | Train Loss Unweighted: 3.2974 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.9409 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1559 | Regression_eta: 0.0112 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7751 | Classification_binary: 2.9409 | Regression_cos_phi: 0.5958 | Regression_energy: 1.7041 | Regression_eta: 0.4979 | Regression_pt: 1.8012 | Regression_sin_phi: 0.7656 | LR: 5.25e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3061s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0691 | max_forward: 0.0145 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3178 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4584.5 | uss_MB: 4106.8 | child_rss_MB: 3197.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:29:30,343] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:41,151] INFO : Step 26000/50000 rank0 | Train Loss: 8.5188 | Train Loss Unweighted: 3.0692 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7322 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1482 | Regression_eta: 0.0131 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8342 | Classification_binary: 2.7322 | Regression_cos_phi: 0.4715 | Regression_energy: 1.6199 | Regression_eta: 0.5842 | Regression_pt: 1.6616 | Regression_sin_phi: 0.6154 | LR: 5.22e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0214 | max_backward: 0.0344 | max_optimizer_step: 0.1205 | max_tensorboard_logging: 0.3825 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4584.5 | uss_MB: 4106.8 | child_rss_MB: 3198.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:29:41,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:41,384] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 12:29:41,573] INFO : Running validation on rank0 +rank0 [2026-10-28 12:29:41,573] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:29:41,573] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:29:42,659] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 12:29:42,805] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:29:42,939] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:29:43,014] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:29:43,087] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:29:43,156] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:29:43,227] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:29:43,297] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:29:43,404] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:29:43,536] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:29:43,668] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:29:43,742] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:29:43,814] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:29:43,883] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:29:43,954] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:29:44,024] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:29:44,128] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:29:44,135] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:29:44,138] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:29:44,141] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:29:44,142] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:29:44,144] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:29:44,147] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:29:44,148] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:29:44,151] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:29:44,154] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:29:44,157] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:29:44,159] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:29:44,160] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:29:44,163] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:29:44,165] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:29:44,166] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:29:50,222] INFO : VALIDATION | Step=26000/50000 | Train Loss=8.5188 | Valid Loss=8.6956 | Train Loss Unweighted=3.0692 | Valid Loss Unweighted=3.1324 | Stale=0 | ETA=71.3m +rank0 [2026-10-28 12:29:50,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:29:50,288] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:29:50,291] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:29:50,458] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:29:52,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:29:52,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:29:53,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:29:53,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:29:53,884] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:29:53,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:29:53,942] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:29:53,944] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:29:54,046] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:29:55,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:29:56,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:29:56,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:29:57,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:29:57,509] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:29:57,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:29:57,566] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:29:57,569] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:29:57,676] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:29:59,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:29:59,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:30:00,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:30:00,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:30:00,945] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:30:00,952] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:30:01,219] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:30:01,630] INFO : Plotted number of elements +rank0 [2026-10-28 12:30:03,077] INFO : Plotted jets +rank0 [2026-10-28 12:30:04,170] INFO : Plotted jet ratio +rank0 [2026-10-28 12:30:04,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:30:04,439] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:30:04,830] INFO : Plotted number of elements +rank0 [2026-10-28 12:30:06,703] INFO : Plotted jets +rank0 [2026-10-28 12:30:07,803] INFO : Plotted jet ratio +rank0 [2026-10-28 12:30:07,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 12:30:08,048] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 12:30:08,411] INFO : Plotted number of elements +rank0 [2026-10-28 12:30:09,790] INFO : Plotted jets +rank0 [2026-10-28 12:30:10,840] INFO : Plotted jet ratio +rank0 [2026-10-28 12:30:22,007] INFO : Step 26100/50000 rank0 | Train Loss: 8.7575 | Train Loss Unweighted: 3.2024 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8523 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1534 | Regression_eta: 0.0110 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7623 | Classification_binary: 2.8523 | Regression_cos_phi: 0.4162 | Regression_energy: 1.6772 | Regression_eta: 0.4896 | Regression_pt: 1.7933 | Regression_sin_phi: 0.7666 | LR: 5.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2547s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0828 | max_optimizer_step: 0.1207 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4773.1 | uss_MB: 4296.6 | child_rss_MB: 3219.8 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:22,222] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:33,164] INFO : Step 26200/50000 rank0 | Train Loss: 8.4734 | Train Loss Unweighted: 3.0377 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6827 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1588 | Regression_eta: 0.0102 | Regression_pt: 0.1543 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6693 | Classification_binary: 2.6827 | Regression_cos_phi: 0.4549 | Regression_energy: 1.7365 | Regression_eta: 0.4569 | Regression_pt: 1.8340 | Regression_sin_phi: 0.6391 | LR: 5.16e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0189 | max_loss: 0.0034 | max_backward: 0.0616 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4773.1 | uss_MB: 4298.7 | child_rss_MB: 3220.1 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:30:33,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:44,167] INFO : Step 26300/50000 rank0 | Train Loss: 9.2084 | Train Loss Unweighted: 3.1564 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7777 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1683 | Regression_eta: 0.0161 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7111 | Classification_binary: 2.7777 | Regression_cos_phi: 0.5491 | Regression_energy: 1.8404 | Regression_eta: 0.7194 | Regression_pt: 1.8926 | Regression_sin_phi: 0.7181 | LR: 5.13e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3073s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4773.1 | uss_MB: 4296.7 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:30:44,391] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:55,034] INFO : Step 26400/50000 rank0 | Train Loss: 8.4618 | Train Loss Unweighted: 3.1612 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.8185 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1494 | Regression_eta: 0.0114 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8176 | Classification_binary: 2.8185 | Regression_cos_phi: 0.3808 | Regression_energy: 1.6334 | Regression_eta: 0.5072 | Regression_pt: 1.7657 | Regression_sin_phi: 0.5385 | LR: 5.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2688s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0151 | max_forward: 0.0143 | max_loss: 0.0032 | max_backward: 0.0327 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.3066 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4773.1 | uss_MB: 4296.8 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:55,249] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:05,995] INFO : Step 26500/50000 rank0 | Train Loss: 9.1826 | Train Loss Unweighted: 3.1310 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7751 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1527 | Regression_eta: 0.0213 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7348 | Classification_binary: 2.7751 | Regression_cos_phi: 0.5025 | Regression_energy: 1.6696 | Regression_eta: 0.9495 | Regression_pt: 1.7343 | Regression_sin_phi: 0.8168 | LR: 5.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4090s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0169 | max_loss: 0.0821 | max_backward: 0.0382 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3855 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4773.1 | uss_MB: 4298.8 | child_rss_MB: 3197.2 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:31:06,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:17,192] INFO : Step 26600/50000 rank0 | Train Loss: 8.8570 | Train Loss Unweighted: 3.1231 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7779 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1510 | Regression_eta: 0.0146 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7589 | Classification_binary: 2.7779 | Regression_cos_phi: 0.5587 | Regression_energy: 1.6511 | Regression_eta: 0.6522 | Regression_pt: 1.6960 | Regression_sin_phi: 0.7622 | LR: 5.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0684 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3979 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4655.2 | uss_MB: 4178.7 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:17,404] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:28,313] INFO : Step 26700/50000 rank0 | Train Loss: 9.5326 | Train Loss Unweighted: 3.2361 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.8431 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1746 | Regression_eta: 0.0164 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8320 | Classification_binary: 2.8431 | Regression_cos_phi: 0.5391 | Regression_energy: 1.9091 | Regression_eta: 0.7300 | Regression_pt: 1.9476 | Regression_sin_phi: 0.7317 | LR: 5.02e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0814 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1186 | max_tensorboard_logging: 0.3166 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4655.2 | uss_MB: 4179.8 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:28,528] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:39,238] INFO : Step 26800/50000 rank0 | Train Loss: 8.1123 | Train Loss Unweighted: 3.0347 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.7095 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1446 | Regression_eta: 0.0097 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7193 | Classification_binary: 2.7095 | Regression_cos_phi: 0.4018 | Regression_energy: 1.5809 | Regression_eta: 0.4304 | Regression_pt: 1.6531 | Regression_sin_phi: 0.6173 | LR: 4.99e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3805s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0836 | max_backward: 0.0329 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4655.2 | uss_MB: 4180.0 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:39,448] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:50,323] INFO : Step 26900/50000 rank0 | Train Loss: 8.9591 | Train Loss Unweighted: 3.1512 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.7565 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1777 | Regression_eta: 0.0092 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8104 | Classification_binary: 2.7565 | Regression_cos_phi: 0.4106 | Regression_energy: 1.9432 | Regression_eta: 0.4115 | Regression_pt: 2.0664 | Regression_sin_phi: 0.5605 | LR: 4.96e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0870 | max_backward: 0.0328 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.3701 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4655.2 | uss_MB: 4179.0 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:50,536] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:01,233] INFO : Step 27000/50000 rank0 | Train Loss: 8.5175 | Train Loss Unweighted: 3.1232 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7843 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1484 | Regression_eta: 0.0100 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7345 | Classification_binary: 2.7843 | Regression_cos_phi: 0.5530 | Regression_energy: 1.6225 | Regression_eta: 0.4469 | Regression_pt: 1.7311 | Regression_sin_phi: 0.6451 | LR: 4.94e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.2192s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.0485 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4656.2 | uss_MB: 4180.1 | child_rss_MB: 3217.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:01,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:01,464] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 12:32:01,545] INFO : Running validation on rank0 +rank0 [2026-10-28 12:32:01,545] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:32:01,545] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:32:02,759] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 12:32:02,905] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:32:03,036] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:32:03,110] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:32:03,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:32:03,246] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:32:03,316] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:32:03,387] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:32:03,496] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:32:03,628] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:32:03,759] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:32:03,829] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:32:03,899] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:32:03,968] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:32:04,040] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:32:04,113] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:32:04,230] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:32:04,238] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:32:04,240] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:32:04,243] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:32:04,244] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:32:04,247] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:32:04,250] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:32:04,251] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:32:04,254] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:32:04,257] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:32:04,257] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:32:04,260] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:32:04,261] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:32:04,264] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:32:04,266] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:32:04,267] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:32:10,079] INFO : VALIDATION | Step=27000/50000 | Train Loss=8.5175 | Valid Loss=8.8705 | Train Loss Unweighted=3.1232 | Valid Loss Unweighted=3.1739 | Stale=1 | ETA=67.8m +rank0 [2026-10-28 12:32:10,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:10,155] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:32:10,158] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:32:10,321] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:11,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:12,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:13,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:13,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:13,710] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:32:13,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:13,769] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:32:13,772] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:13,877] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:15,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:15,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:16,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:17,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:17,219] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:32:17,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:17,276] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:32:17,279] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:32:17,393] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:18,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:19,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:19,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:20,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:20,461] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:32:20,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:32:20,730] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:32:21,140] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:22,579] INFO : Plotted jets +rank0 [2026-10-28 12:32:23,662] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:23,693] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:23,928] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:24,319] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:26,193] INFO : Plotted jets +rank0 [2026-10-28 12:32:27,294] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:27,326] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 12:32:27,533] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 12:32:27,899] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:29,372] INFO : Plotted jets +rank0 [2026-10-28 12:32:30,430] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:41,586] INFO : Step 27100/50000 rank0 | Train Loss: 8.6993 | Train Loss Unweighted: 3.0721 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7215 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1543 | Regression_eta: 0.0118 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7528 | Classification_binary: 2.7215 | Regression_cos_phi: 0.4793 | Regression_energy: 1.6865 | Regression_eta: 0.5247 | Regression_pt: 1.7713 | Regression_sin_phi: 0.7632 | LR: 4.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4073s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1012 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4852.7 | uss_MB: 4377.1 | child_rss_MB: 3196.2 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:32:41,799] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:52,587] INFO : Step 27200/50000 rank0 | Train Loss: 8.2814 | Train Loss Unweighted: 3.0758 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.7533 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1407 | Regression_eta: 0.0112 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6688 | Classification_binary: 2.7533 | Regression_cos_phi: 0.4863 | Regression_energy: 1.5385 | Regression_eta: 0.5011 | Regression_pt: 1.6394 | Regression_sin_phi: 0.6940 | LR: 4.88e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3757s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0577 | max_backward: 0.0341 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3745 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4852.7 | uss_MB: 4377.1 | child_rss_MB: 3197.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:32:52,804] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:03,690] INFO : Step 27300/50000 rank0 | Train Loss: 9.0032 | Train Loss Unweighted: 3.2906 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.9190 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1670 | Regression_eta: 0.0160 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7181 | Classification_binary: 2.9190 | Regression_cos_phi: 0.4247 | Regression_energy: 1.8260 | Regression_eta: 0.7114 | Regression_pt: 1.8689 | Regression_sin_phi: 0.5349 | LR: 4.85e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4052s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0982 | max_backward: 0.0335 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4852.7 | uss_MB: 4377.1 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:33:03,901] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:14,887] INFO : Step 27400/50000 rank0 | Train Loss: 8.3476 | Train Loss Unweighted: 3.0878 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7612 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1428 | Regression_eta: 0.0109 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7627 | Classification_binary: 2.7612 | Regression_cos_phi: 0.4945 | Regression_energy: 1.5616 | Regression_eta: 0.4841 | Regression_pt: 1.6436 | Regression_sin_phi: 0.6399 | LR: 4.82e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1214 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4852.7 | uss_MB: 4377.2 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:33:15,102] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:25,989] INFO : Step 27500/50000 rank0 | Train Loss: 8.9350 | Train Loss Unweighted: 3.1282 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7858 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1473 | Regression_eta: 0.0160 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7247 | Classification_binary: 2.7858 | Regression_cos_phi: 0.5819 | Regression_energy: 1.6108 | Regression_eta: 0.7155 | Regression_pt: 1.6868 | Regression_sin_phi: 0.8295 | LR: 4.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3617s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0468 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3795 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4852.7 | uss_MB: 4377.3 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:33:26,204] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:37,118] INFO : Step 27600/50000 rank0 | Train Loss: 8.0661 | Train Loss Unweighted: 2.9602 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.6369 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1425 | Regression_eta: 0.0084 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7787 | Classification_binary: 2.6369 | Regression_cos_phi: 0.4724 | Regression_energy: 1.5578 | Regression_eta: 0.3745 | Regression_pt: 1.6401 | Regression_sin_phi: 0.6057 | LR: 4.77e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4227s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1005 | max_loss: 0.0031 | max_backward: 0.0331 | max_optimizer_step: 0.1211 | max_tensorboard_logging: 0.3717 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4852.7 | uss_MB: 4377.3 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:33:37,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:48,221] INFO : Step 27700/50000 rank0 | Train Loss: 7.9794 | Train Loss Unweighted: 2.9328 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6192 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1364 | Regression_eta: 0.0104 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7312 | Classification_binary: 2.6192 | Regression_cos_phi: 0.4545 | Regression_energy: 1.4914 | Regression_eta: 0.4656 | Regression_pt: 1.5870 | Regression_sin_phi: 0.6305 | LR: 4.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4247s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1205 | max_loss: 0.0031 | max_backward: 0.0342 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3809 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4728.8 | uss_MB: 4254.5 | child_rss_MB: 3194.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:48,439] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:33:59,236] INFO : Step 27800/50000 rank0 | Train Loss: 8.6637 | Train Loss Unweighted: 3.1647 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8083 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1576 | Regression_eta: 0.0108 | Regression_pt: 0.1549 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7473 | Classification_binary: 2.8083 | Regression_cos_phi: 0.4251 | Regression_energy: 1.7225 | Regression_eta: 0.4800 | Regression_pt: 1.8407 | Regression_sin_phi: 0.6397 | LR: 4.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.1048 | max_backward: 0.0336 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3749 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4728.8 | uss_MB: 4254.6 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:59,449] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:10,131] INFO : Step 27900/50000 rank0 | Train Loss: 8.0670 | Train Loss Unweighted: 2.9797 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.6609 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1438 | Regression_eta: 0.0094 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7385 | Classification_binary: 2.6609 | Regression_cos_phi: 0.4685 | Regression_energy: 1.5726 | Regression_eta: 0.4180 | Regression_pt: 1.5682 | Regression_sin_phi: 0.6402 | LR: 4.68e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2745s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0033 | max_backward: 0.0905 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2833 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4728.8 | uss_MB: 4254.6 | child_rss_MB: 3221.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:34:10,344] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:21,244] INFO : Step 28000/50000 rank0 | Train Loss: 8.6490 | Train Loss Unweighted: 3.1135 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.7583 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1578 | Regression_eta: 0.0107 | Regression_pt: 0.1522 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 2.7583 | Regression_cos_phi: 0.4548 | Regression_energy: 1.7247 | Regression_eta: 0.4767 | Regression_pt: 1.8079 | Regression_sin_phi: 0.6418 | LR: 4.66e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3279s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0032 | max_backward: 0.1418 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2740 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4728.8 | uss_MB: 4254.7 | child_rss_MB: 3220.4 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:34:21,458] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:21,477] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 12:34:21,560] INFO : Running validation on rank0 +rank0 [2026-10-28 12:34:21,560] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:34:21,560] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:34:22,757] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 12:34:22,902] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:34:23,035] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:34:23,109] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:34:23,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:34:23,247] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:34:23,316] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:34:23,388] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:34:23,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:34:23,629] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:34:23,760] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:34:23,832] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:34:23,901] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:34:23,969] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:34:24,039] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:34:24,111] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:34:24,217] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:34:24,224] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:34:24,226] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:34:24,227] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:34:24,230] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:34:24,233] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:34:24,234] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:34:24,236] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:34:24,238] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:34:24,240] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:34:24,243] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:34:24,246] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:34:24,247] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:34:24,249] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:34:24,252] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:34:24,253] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:34:30,108] INFO : VALIDATION | Step=28000/50000 | Train Loss=8.6490 | Valid Loss=8.7492 | Train Loss Unweighted=3.1135 | Valid Loss Unweighted=3.1402 | Stale=2 | ETA=64.4m +rank0 [2026-10-28 12:34:30,118] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:30,174] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:34:30,178] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:30,275] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:31,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:32,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:32,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:33,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:33,765] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:34:33,769] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:33,835] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:34:33,838] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:33,932] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:35,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:36,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:36,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:37,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:37,318] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:34:37,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:37,376] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:34:37,378] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:34:37,484] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:39,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:39,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:40,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:40,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:40,802] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:34:40,810] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:41,075] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:41,484] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:42,918] INFO : Plotted jets +rank0 [2026-10-28 12:34:44,000] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:44,032] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:44,267] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:44,662] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:46,585] INFO : Plotted jets +rank0 [2026-10-28 12:34:47,690] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:47,722] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:47,935] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:48,299] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:49,674] INFO : Plotted jets +rank0 [2026-10-28 12:34:50,729] INFO : Plotted jet ratio +rank0 [2026-10-28 12:35:01,940] INFO : Step 28100/50000 rank0 | Train Loss: 8.4413 | Train Loss Unweighted: 3.0659 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7427 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1400 | Regression_eta: 0.0146 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7446 | Classification_binary: 2.7427 | Regression_cos_phi: 0.3990 | Regression_energy: 1.5304 | Regression_eta: 0.6524 | Regression_pt: 1.5978 | Regression_sin_phi: 0.7744 | LR: 4.63e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4502s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1242 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4928.2 | uss_MB: 4453.9 | child_rss_MB: 3198.9 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:35:02,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:13,008] INFO : Step 28200/50000 rank0 | Train Loss: 8.8933 | Train Loss Unweighted: 3.0885 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.7558 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1397 | Regression_eta: 0.0175 | Regression_pt: 0.1358 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8403 | Classification_binary: 2.7558 | Regression_cos_phi: 0.5929 | Regression_energy: 1.5271 | Regression_eta: 0.7783 | Regression_pt: 1.6136 | Regression_sin_phi: 0.7853 | LR: 4.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0166 | max_loss: 0.0425 | max_backward: 0.0382 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.4274 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4928.2 | uss_MB: 4456.0 | child_rss_MB: 3199.3 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:35:13,219] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:24,198] INFO : Step 28300/50000 rank0 | Train Loss: 9.2510 | Train Loss Unweighted: 3.1837 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.8101 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1647 | Regression_eta: 0.0172 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7424 | Classification_binary: 2.8101 | Regression_cos_phi: 0.4889 | Regression_energy: 1.8004 | Regression_eta: 0.7686 | Regression_pt: 1.8556 | Regression_sin_phi: 0.7850 | LR: 4.57e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4577s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0984 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.3918 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4928.2 | uss_MB: 4454.0 | child_rss_MB: 3197.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:35:24,413] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:35,287] INFO : Step 28400/50000 rank0 | Train Loss: 8.4454 | Train Loss Unweighted: 3.0744 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7281 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1528 | Regression_eta: 0.0097 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7118 | Classification_binary: 2.7281 | Regression_cos_phi: 0.4346 | Regression_energy: 1.6710 | Regression_eta: 0.4321 | Regression_pt: 1.7947 | Regression_sin_phi: 0.6731 | LR: 4.55e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2806s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0279 | max_forward: 0.0167 | max_loss: 0.0037 | max_backward: 0.0342 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4928.2 | uss_MB: 4454.0 | child_rss_MB: 3203.6 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:35:35,499] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:46,378] INFO : Step 28500/50000 rank0 | Train Loss: 8.7826 | Train Loss Unweighted: 3.0968 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.7558 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1469 | Regression_eta: 0.0175 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6823 | Classification_binary: 2.7558 | Regression_cos_phi: 0.5985 | Regression_energy: 1.6060 | Regression_eta: 0.7798 | Regression_pt: 1.6890 | Regression_sin_phi: 0.6711 | LR: 4.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3950s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0792 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3803 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4928.2 | uss_MB: 4454.1 | child_rss_MB: 3199.6 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:35:46,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:57,416] INFO : Step 28600/50000 rank0 | Train Loss: 8.4523 | Train Loss Unweighted: 3.0593 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.7246 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1474 | Regression_eta: 0.0111 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6942 | Classification_binary: 2.7246 | Regression_cos_phi: 0.4386 | Regression_energy: 1.6112 | Regression_eta: 0.4940 | Regression_pt: 1.6939 | Regression_sin_phi: 0.7957 | LR: 4.49e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4280s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0573 | max_backward: 0.0335 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3885 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4928.2 | uss_MB: 4454.0 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:35:57,631] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:08,599] INFO : Step 28700/50000 rank0 | Train Loss: 8.4228 | Train Loss Unweighted: 3.0729 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.7447 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1423 | Regression_eta: 0.0152 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7434 | Classification_binary: 2.7447 | Regression_cos_phi: 0.4148 | Regression_energy: 1.5561 | Regression_eta: 0.6795 | Regression_pt: 1.6350 | Regression_sin_phi: 0.6493 | LR: 4.46e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4101s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1006 | max_loss: 0.0036 | max_backward: 0.0328 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3821 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4803.7 | uss_MB: 4330.4 | child_rss_MB: 3201.4 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:36:08,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:19,703] INFO : Step 28800/50000 rank0 | Train Loss: 8.8311 | Train Loss Unweighted: 3.1146 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7493 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1622 | Regression_eta: 0.0128 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7268 | Classification_binary: 2.7493 | Regression_cos_phi: 0.4707 | Regression_energy: 1.7734 | Regression_eta: 0.5709 | Regression_pt: 1.8601 | Regression_sin_phi: 0.6799 | LR: 4.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4106s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0935 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1217 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4803.7 | uss_MB: 4330.5 | child_rss_MB: 3197.2 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:36:19,918] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:30,505] INFO : Step 28900/50000 rank0 | Train Loss: 9.2975 | Train Loss Unweighted: 3.2516 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8664 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1722 | Regression_eta: 0.0141 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7252 | Classification_binary: 2.8664 | Regression_cos_phi: 0.5357 | Regression_energy: 1.8830 | Regression_eta: 0.6305 | Regression_pt: 1.9457 | Regression_sin_phi: 0.7109 | LR: 4.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0990 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4803.7 | uss_MB: 4330.5 | child_rss_MB: 3226.5 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:36:30,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:41,611] INFO : Step 29000/50000 rank0 | Train Loss: 8.6444 | Train Loss Unweighted: 3.1025 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.7444 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1584 | Regression_eta: 0.0098 | Regression_pt: 0.1555 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7700 | Classification_binary: 2.7444 | Regression_cos_phi: 0.4613 | Regression_energy: 1.7314 | Regression_eta: 0.4361 | Regression_pt: 1.8479 | Regression_sin_phi: 0.6533 | LR: 4.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0165 | max_loss: 0.0034 | max_backward: 0.1450 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4803.7 | uss_MB: 4333.1 | child_rss_MB: 3225.7 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:36:41,825] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:41,843] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 12:36:41,926] INFO : Running validation on rank0 +rank0 [2026-10-28 12:36:41,926] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:36:41,927] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:36:43,144] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 12:36:43,289] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:36:43,421] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:36:43,495] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:36:43,564] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:36:43,632] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:36:43,704] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:36:43,774] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:36:43,883] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:36:44,015] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:36:44,145] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:36:44,217] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:36:44,286] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:36:44,355] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:36:44,425] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:36:44,495] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:36:44,599] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:36:44,606] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:36:44,609] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:36:44,612] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:36:44,614] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:36:44,616] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:36:44,620] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:36:44,622] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:36:44,623] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:36:44,626] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:36:44,629] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:36:44,630] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:36:44,632] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:36:44,635] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:36:44,636] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:36:44,639] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:36:50,505] INFO : VALIDATION | Step=29000/50000 | Train Loss=8.6444 | Valid Loss=8.4959 | Train Loss Unweighted=3.1025 | Valid Loss Unweighted=3.0585 | Stale=0 | ETA=61.0m +rank0 [2026-10-28 12:36:50,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:36:50,572] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:36:50,576] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:36:50,745] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:36:52,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:36:52,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:36:53,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:36:54,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:36:54,162] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:36:54,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:36:54,220] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:36:54,223] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:36:54,334] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:36:55,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:36:56,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:36:56,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:36:57,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:36:57,554] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:36:57,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:36:57,611] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:36:57,614] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:36:57,781] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:36:59,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:36:59,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:37:00,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:37:00,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:37:00,977] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:37:00,985] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:37:01,242] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:37:01,651] INFO : Plotted number of elements +rank0 [2026-10-28 12:37:03,099] INFO : Plotted jets +rank0 [2026-10-28 12:37:04,185] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:04,221] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:37:04,460] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:37:04,852] INFO : Plotted number of elements +rank0 [2026-10-28 12:37:06,736] INFO : Plotted jets +rank0 [2026-10-28 12:37:07,843] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:07,877] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 12:37:08,085] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 12:37:08,454] INFO : Plotted number of elements +rank0 [2026-10-28 12:37:09,836] INFO : Plotted jets +rank0 [2026-10-28 12:37:10,897] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:22,135] INFO : Step 29100/50000 rank0 | Train Loss: 8.1597 | Train Loss Unweighted: 3.0292 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7121 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1384 | Regression_eta: 0.0106 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7109 | Classification_binary: 2.7121 | Regression_cos_phi: 0.4648 | Regression_energy: 1.5132 | Regression_eta: 0.4743 | Regression_pt: 1.6018 | Regression_sin_phi: 0.6825 | LR: 4.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4221s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0158 | max_loss: 0.0898 | max_backward: 0.0338 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4996.4 | uss_MB: 4523.5 | child_rss_MB: 3205.3 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:37:22,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:33,371] INFO : Step 29200/50000 rank0 | Train Loss: 8.4643 | Train Loss Unweighted: 3.1143 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7650 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1568 | Regression_eta: 0.0100 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7246 | Classification_binary: 2.7650 | Regression_cos_phi: 0.4444 | Regression_energy: 1.7140 | Regression_eta: 0.4474 | Regression_pt: 1.7846 | Regression_sin_phi: 0.5842 | LR: 4.33e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0185 | max_forward: 0.0146 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2994 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4996.4 | uss_MB: 4523.5 | child_rss_MB: 3207.3 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:33,584] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:44,438] INFO : Step 29300/50000 rank0 | Train Loss: 8.4119 | Train Loss Unweighted: 3.0736 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7302 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1512 | Regression_eta: 0.0107 | Regression_pt: 0.1489 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7303 | Classification_binary: 2.7302 | Regression_cos_phi: 0.4200 | Regression_energy: 1.6528 | Regression_eta: 0.4776 | Regression_pt: 1.7696 | Regression_sin_phi: 0.6314 | LR: 4.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3558s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0427 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.3764 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:44,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:55,556] INFO : Step 29400/50000 rank0 | Train Loss: 8.3027 | Train Loss Unweighted: 3.0531 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7216 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1449 | Regression_eta: 0.0084 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7606 | Classification_binary: 2.7216 | Regression_cos_phi: 0.5280 | Regression_energy: 1.5837 | Regression_eta: 0.3729 | Regression_pt: 1.7009 | Regression_sin_phi: 0.6349 | LR: 4.27e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4414s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1097 | max_loss: 0.0035 | max_backward: 0.0342 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3203.6 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:55,768] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:06,648] INFO : Step 29500/50000 rank0 | Train Loss: 8.2841 | Train Loss Unweighted: 2.9511 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6207 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1439 | Regression_eta: 0.0090 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7643 | Classification_binary: 2.6207 | Regression_cos_phi: 0.4873 | Regression_energy: 1.5736 | Regression_eta: 0.4000 | Regression_pt: 1.6838 | Regression_sin_phi: 0.7544 | LR: 4.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3404s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1056 | max_forward: 0.0148 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:38:06,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:17,600] INFO : Step 29600/50000 rank0 | Train Loss: 8.3190 | Train Loss Unweighted: 3.0665 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7338 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1463 | Regression_eta: 0.0101 | Regression_pt: 0.1425 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7529 | Classification_binary: 2.7338 | Regression_cos_phi: 0.4659 | Regression_energy: 1.5993 | Regression_eta: 0.4520 | Regression_pt: 1.6934 | Regression_sin_phi: 0.6218 | LR: 4.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3825s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0747 | max_backward: 0.0332 | max_optimizer_step: 0.1215 | max_tensorboard_logging: 0.3699 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:38:17,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:28,820] INFO : Step 29700/50000 rank0 | Train Loss: 9.1576 | Train Loss Unweighted: 3.2718 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8855 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1727 | Regression_eta: 0.0118 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7282 | Classification_binary: 2.8855 | Regression_cos_phi: 0.4942 | Regression_energy: 1.8878 | Regression_eta: 0.5250 | Regression_pt: 1.9980 | Regression_sin_phi: 0.6389 | LR: 4.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0307 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1228 | max_tensorboard_logging: 0.3704 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4875.3 | uss_MB: 4404.7 | child_rss_MB: 3202.9 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:38:29,036] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:39,779] INFO : Step 29800/50000 rank0 | Train Loss: 7.7644 | Train Loss Unweighted: 3.0393 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7267 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1390 | Regression_eta: 0.0076 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7120 | Classification_binary: 2.7267 | Regression_cos_phi: 0.3251 | Regression_energy: 1.5191 | Regression_eta: 0.3368 | Regression_pt: 1.6242 | Regression_sin_phi: 0.5204 | LR: 4.16e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3984s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0900 | max_backward: 0.0334 | max_optimizer_step: 0.1219 | max_tensorboard_logging: 0.3746 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4875.3 | uss_MB: 4403.0 | child_rss_MB: 3200.5 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:38:39,992] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:51,024] INFO : Step 29900/50000 rank0 | Train Loss: 9.5750 | Train Loss Unweighted: 3.2311 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.8528 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1658 | Regression_eta: 0.0196 | Regression_pt: 0.1554 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7178 | Classification_binary: 2.8528 | Regression_cos_phi: 0.5789 | Regression_energy: 1.8127 | Regression_eta: 0.8754 | Regression_pt: 1.8464 | Regression_sin_phi: 0.8910 | LR: 4.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3413s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.1078 | max_forward: 0.0170 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1224 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4875.3 | uss_MB: 4405.4 | child_rss_MB: 3201.0 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:38:51,239] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:02,127] INFO : Step 30000/50000 rank0 | Train Loss: 8.9942 | Train Loss Unweighted: 3.0947 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.7383 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1578 | Regression_eta: 0.0135 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7222 | Classification_binary: 2.7383 | Regression_cos_phi: 0.5116 | Regression_energy: 1.7252 | Regression_eta: 0.6036 | Regression_pt: 1.7610 | Regression_sin_phi: 0.9323 | LR: 4.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3900s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0807 | max_loss: 0.0035 | max_backward: 0.0332 | max_optimizer_step: 0.1230 | max_tensorboard_logging: 0.3752 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4875.3 | uss_MB: 4403.3 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:39:02,342] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:02,402] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 12:39:02,483] INFO : Running validation on rank0 +rank0 [2026-10-28 12:39:02,484] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:39:02,484] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:39:03,584] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 12:39:03,731] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:39:03,864] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:39:03,938] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:39:04,007] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:39:04,074] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:39:04,144] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:39:04,214] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:39:04,323] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:39:04,455] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:39:04,587] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:39:04,658] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:39:04,728] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:39:04,797] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:39:04,867] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:39:04,937] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:39:05,041] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:39:05,048] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:39:05,051] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:39:05,054] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:39:05,055] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:39:05,057] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:39:05,060] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:39:05,062] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:39:05,064] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:39:05,067] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:39:05,068] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:39:05,070] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:39:05,073] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:39:05,074] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:39:05,077] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:39:05,079] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:39:11,045] INFO : VALIDATION | Step=30000/50000 | Train Loss=8.9942 | Valid Loss=8.6269 | Train Loss Unweighted=3.0947 | Valid Loss Unweighted=3.1125 | Stale=1 | ETA=57.8m +rank0 [2026-10-28 12:39:11,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:11,109] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:39:11,112] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:39:11,278] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:12,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:13,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:14,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:14,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:14,926] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:39:14,930] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:14,986] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:39:14,989] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:15,099] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:16,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:17,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:17,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:18,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:18,439] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:39:18,443] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:18,495] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:39:18,498] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:39:18,602] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:20,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:20,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:21,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:21,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:21,797] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:39:21,805] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:39:22,065] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:39:22,474] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:23,921] INFO : Plotted jets +rank0 [2026-10-28 12:39:25,006] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:25,039] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:25,272] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:25,664] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:27,546] INFO : Plotted jets +rank0 [2026-10-28 12:39:28,644] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:28,676] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 12:39:28,886] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 12:39:29,249] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:30,630] INFO : Plotted jets +rank0 [2026-10-28 12:39:31,682] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:43,039] INFO : Step 30100/50000 rank0 | Train Loss: 8.4683 | Train Loss Unweighted: 3.1075 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.7584 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1541 | Regression_eta: 0.0105 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7982 | Classification_binary: 2.7584 | Regression_cos_phi: 0.4948 | Regression_energy: 1.6850 | Regression_eta: 0.4685 | Regression_pt: 1.7897 | Regression_sin_phi: 0.4736 | LR: 4.08e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4512s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1075 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.3957 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4944.1 | uss_MB: 4474.1 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:39:43,251] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:54,084] INFO : Step 30200/50000 rank0 | Train Loss: 8.5808 | Train Loss Unweighted: 2.9869 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6554 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1454 | Regression_eta: 0.0198 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7235 | Classification_binary: 2.6554 | Regression_cos_phi: 0.4684 | Regression_energy: 1.5895 | Regression_eta: 0.8845 | Regression_pt: 1.5748 | Regression_sin_phi: 0.6845 | LR: 4.06e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4457s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1249 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4944.1 | uss_MB: 4474.9 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:54,299] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:05,158] INFO : Step 30300/50000 rank0 | Train Loss: 8.8846 | Train Loss Unweighted: 3.2142 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8558 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1585 | Regression_eta: 0.0167 | Regression_pt: 0.1507 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7280 | Classification_binary: 2.8558 | Regression_cos_phi: 0.4490 | Regression_energy: 1.7327 | Regression_eta: 0.7452 | Regression_pt: 1.7910 | Regression_sin_phi: 0.5829 | LR: 4.03e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4134s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1194 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1205 | max_tensorboard_logging: 0.3697 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4944.1 | uss_MB: 4475.3 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:05,372] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:16,255] INFO : Step 30400/50000 rank0 | Train Loss: 9.2181 | Train Loss Unweighted: 3.0798 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7084 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1624 | Regression_eta: 0.0238 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7928 | Classification_binary: 2.7084 | Regression_cos_phi: 0.4619 | Regression_energy: 1.7752 | Regression_eta: 1.0631 | Regression_pt: 1.7867 | Regression_sin_phi: 0.6299 | LR: 4.00e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.4029s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1107 | max_loss: 0.0033 | max_backward: 0.0342 | max_optimizer_step: 0.1228 | max_tensorboard_logging: 0.3619 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4944.1 | uss_MB: 4475.4 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:16,467] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:27,281] INFO : Step 30500/50000 rank0 | Train Loss: 9.0331 | Train Loss Unweighted: 3.0008 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6496 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1489 | Regression_eta: 0.0258 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7171 | Classification_binary: 2.6496 | Regression_cos_phi: 0.4712 | Regression_energy: 1.6277 | Regression_eta: 1.1507 | Regression_pt: 1.6940 | Regression_sin_phi: 0.7228 | LR: 3.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0147 | max_backward: 0.0339 | max_optimizer_step: 0.1235 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4944.1 | uss_MB: 4475.4 | child_rss_MB: 3197.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:27,492] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:38,496] INFO : Step 30600/50000 rank0 | Train Loss: 8.3329 | Train Loss Unweighted: 3.0949 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7635 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1450 | Regression_eta: 0.0110 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7659 | Classification_binary: 2.7635 | Regression_cos_phi: 0.4576 | Regression_energy: 1.5848 | Regression_eta: 0.4911 | Regression_pt: 1.6858 | Regression_sin_phi: 0.5842 | LR: 3.95e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3332s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0645 | max_forward: 0.0145 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1221 | max_tensorboard_logging: 0.3200 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4944.1 | uss_MB: 4475.5 | child_rss_MB: 3200.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:40:38,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:49,599] INFO : Step 30700/50000 rank0 | Train Loss: 9.2003 | Train Loss Unweighted: 3.1740 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.8052 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1646 | Regression_eta: 0.0209 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7397 | Classification_binary: 2.8052 | Regression_cos_phi: 0.4903 | Regression_energy: 1.7997 | Regression_eta: 0.9313 | Regression_pt: 1.7714 | Regression_sin_phi: 0.6629 | LR: 3.92e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3326s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0708 | max_forward: 0.0134 | max_loss: 0.0031 | max_backward: 0.0318 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.3120 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4942.9 | uss_MB: 4474.3 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:40:49,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:00,674] INFO : Step 30800/50000 rank0 | Train Loss: 7.9721 | Train Loss Unweighted: 3.0409 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.7356 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1340 | Regression_eta: 0.0108 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7053 | Classification_binary: 2.7356 | Regression_cos_phi: 0.4334 | Regression_energy: 1.4647 | Regression_eta: 0.4802 | Regression_pt: 1.5265 | Regression_sin_phi: 0.6264 | LR: 3.90e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3774s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0638 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3806 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4943.9 | uss_MB: 4475.1 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:41:00,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:11,794] INFO : Step 30900/50000 rank0 | Train Loss: 8.9041 | Train Loss Unweighted: 3.0738 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7171 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1587 | Regression_eta: 0.0165 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7646 | Classification_binary: 2.7171 | Regression_cos_phi: 0.3971 | Regression_energy: 1.7355 | Regression_eta: 0.7354 | Regression_pt: 1.7395 | Regression_sin_phi: 0.8149 | LR: 3.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4040s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0593 | max_loss: 0.0033 | max_backward: 0.0391 | max_optimizer_step: 0.1242 | max_tensorboard_logging: 0.3829 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4944.9 | uss_MB: 4477.8 | child_rss_MB: 3201.2 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:41:12,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:22,863] INFO : Step 31000/50000 rank0 | Train Loss: 8.7065 | Train Loss Unweighted: 3.0753 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.7166 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1575 | Regression_eta: 0.0111 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7718 | Classification_binary: 2.7166 | Regression_cos_phi: 0.4664 | Regression_energy: 1.7215 | Regression_eta: 0.4943 | Regression_pt: 1.8427 | Regression_sin_phi: 0.6932 | LR: 3.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3910s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0634 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4944.9 | uss_MB: 4475.8 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:41:23,078] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:23,114] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 12:41:23,197] INFO : Running validation on rank0 +rank0 [2026-10-28 12:41:23,197] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:41:23,198] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:41:24,278] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 12:41:24,424] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:41:24,556] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:41:24,630] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:41:24,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:41:24,766] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:41:24,836] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:41:24,905] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:41:25,014] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:41:25,148] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:41:25,279] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:41:25,350] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:41:25,420] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:41:25,488] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:41:25,558] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:41:25,629] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:41:25,733] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:41:25,739] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:41:25,742] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:41:25,744] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:41:25,746] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:41:25,749] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:41:25,750] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:41:25,753] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:41:25,756] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:41:25,759] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:41:25,761] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:41:25,762] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:41:25,765] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:41:25,768] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:41:25,770] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:41:25,771] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:41:31,623] INFO : VALIDATION | Step=31000/50000 | Train Loss=8.7065 | Valid Loss=8.5125 | Train Loss Unweighted=3.0753 | Valid Loss Unweighted=3.0729 | Stale=2 | ETA=54.5m +rank0 [2026-10-28 12:41:31,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:31,688] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:41:31,692] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:31,858] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:33,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:34,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:34,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:35,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:35,592] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:35,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:35,650] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:41:35,653] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:35,832] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:37,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:37,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:38,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:39,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:39,178] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:41:39,182] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:39,236] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:41:39,239] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:41:39,344] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:40,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:41,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:41,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:42,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:42,710] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:42,718] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:42,980] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:43,385] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:44,819] INFO : Plotted jets +rank0 [2026-10-28 12:41:45,900] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:45,933] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:46,163] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:46,556] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:48,428] INFO : Plotted jets +rank0 [2026-10-28 12:41:49,522] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:49,556] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:49,767] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:50,127] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:51,498] INFO : Plotted jets +rank0 [2026-10-28 12:41:52,542] INFO : Plotted jet ratio +rank0 [2026-10-28 12:42:03,955] INFO : Step 31100/50000 rank0 | Train Loss: 8.5782 | Train Loss Unweighted: 2.9896 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6411 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1548 | Regression_eta: 0.0166 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6968 | Classification_binary: 2.6411 | Regression_cos_phi: 0.4962 | Regression_energy: 1.6918 | Regression_eta: 0.7424 | Regression_pt: 1.7189 | Regression_sin_phi: 0.5911 | LR: 3.82e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3388s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.2175 | max_tensorboard_logging: 0.2925 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5020.9 | uss_MB: 4548.6 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:04,188] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:15,204] INFO : Step 31200/50000 rank0 | Train Loss: 8.3741 | Train Loss Unweighted: 3.0365 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6959 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1533 | Regression_eta: 0.0106 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7166 | Classification_binary: 2.6959 | Regression_cos_phi: 0.4515 | Regression_energy: 1.6755 | Regression_eta: 0.4724 | Regression_pt: 1.7098 | Regression_sin_phi: 0.6524 | LR: 3.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3568s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.1303 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5021.9 | uss_MB: 4549.6 | child_rss_MB: 3224.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:42:15,416] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:26,281] INFO : Step 31300/50000 rank0 | Train Loss: 8.4267 | Train Loss Unweighted: 3.0419 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6906 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1549 | Regression_eta: 0.0083 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8155 | Classification_binary: 2.6906 | Regression_cos_phi: 0.4539 | Regression_energy: 1.6931 | Regression_eta: 0.3680 | Regression_pt: 1.8219 | Regression_sin_phi: 0.5836 | LR: 3.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3783s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0168 | max_loss: 0.0033 | max_backward: 0.1463 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.2909 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5021.9 | uss_MB: 4552.2 | child_rss_MB: 3216.9 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:42:26,493] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:37,369] INFO : Step 31400/50000 rank0 | Train Loss: 9.0298 | Train Loss Unweighted: 3.0601 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6872 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1656 | Regression_eta: 0.0132 | Regression_pt: 0.1582 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7203 | Classification_binary: 2.6872 | Regression_cos_phi: 0.5304 | Regression_energy: 1.8106 | Regression_eta: 0.5876 | Regression_pt: 1.8793 | Regression_sin_phi: 0.8143 | LR: 3.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3562s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.1342 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5021.9 | uss_MB: 4550.5 | child_rss_MB: 3217.6 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:42:37,581] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:48,371] INFO : Step 31500/50000 rank0 | Train Loss: 8.8985 | Train Loss Unweighted: 3.1530 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.7992 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1534 | Regression_eta: 0.0134 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7383 | Classification_binary: 2.7992 | Regression_cos_phi: 0.4834 | Regression_energy: 1.6773 | Regression_eta: 0.5984 | Regression_pt: 1.7986 | Regression_sin_phi: 0.8033 | LR: 3.71e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5022.9 | uss_MB: 4550.5 | child_rss_MB: 3198.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:48,604] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:59,467] INFO : Step 31600/50000 rank0 | Train Loss: 8.4390 | Train Loss Unweighted: 3.0108 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.6645 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1536 | Regression_eta: 0.0103 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7120 | Classification_binary: 2.6645 | Regression_cos_phi: 0.4754 | Regression_energy: 1.6790 | Regression_eta: 0.4583 | Regression_pt: 1.7705 | Regression_sin_phi: 0.6793 | LR: 3.69e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3002s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5022.9 | uss_MB: 4550.7 | child_rss_MB: 3198.0 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:59,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:10,362] INFO : Step 31700/50000 rank0 | Train Loss: 9.2963 | Train Loss Unweighted: 3.2675 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.8981 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1603 | Regression_eta: 0.0148 | Regression_pt: 0.1566 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7982 | Classification_binary: 2.8981 | Regression_cos_phi: 0.5051 | Regression_energy: 1.7527 | Regression_eta: 0.6614 | Regression_pt: 1.8602 | Regression_sin_phi: 0.8206 | LR: 3.66e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3287s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0512 | max_forward: 0.0163 | max_loss: 0.0033 | max_backward: 0.0381 | max_optimizer_step: 0.1240 | max_tensorboard_logging: 0.3023 | max_tensorboard_flush: 0.0133 | Memory: rss_MB: 5022.2 | uss_MB: 4550.8 | child_rss_MB: 3199.9 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:43:10,578] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:21,450] INFO : Step 31800/50000 rank0 | Train Loss: 7.9462 | Train Loss Unweighted: 2.9626 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.6545 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1331 | Regression_eta: 0.0090 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6807 | Classification_binary: 2.6545 | Regression_cos_phi: 0.4615 | Regression_energy: 1.4555 | Regression_eta: 0.4031 | Regression_pt: 1.5836 | Regression_sin_phi: 0.7073 | LR: 3.64e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4292s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0819 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5023.2 | uss_MB: 4549.9 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:43:21,662] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:32,443] INFO : Step 31900/50000 rank0 | Train Loss: 7.9949 | Train Loss Unweighted: 2.9342 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.6233 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1351 | Regression_eta: 0.0153 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6595 | Classification_binary: 2.6233 | Regression_cos_phi: 0.3513 | Regression_energy: 1.4768 | Regression_eta: 0.6829 | Regression_pt: 1.5524 | Regression_sin_phi: 0.6487 | LR: 3.61e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.1046 | max_backward: 0.0341 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5023.2 | uss_MB: 4550.1 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:43:32,654] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:43,537] INFO : Step 32000/50000 rank0 | Train Loss: 8.4876 | Train Loss Unweighted: 2.9761 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.6478 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1405 | Regression_eta: 0.0199 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6887 | Classification_binary: 2.6478 | Regression_cos_phi: 0.5081 | Regression_energy: 1.5356 | Regression_eta: 0.8874 | Regression_pt: 1.6090 | Regression_sin_phi: 0.6110 | LR: 3.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4162s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0561 | max_backward: 0.0334 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.3866 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5023.2 | uss_MB: 4550.5 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:43:43,749] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:43,768] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 12:43:43,953] INFO : Running validation on rank0 +rank0 [2026-10-28 12:43:43,954] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:43:43,954] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:43:45,033] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 12:43:45,179] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:43:45,311] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:43:45,386] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:43:45,456] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:43:45,523] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:43:45,593] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:43:45,662] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:43:45,771] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:43:45,903] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:43:46,035] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:43:46,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:43:46,178] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:43:46,247] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:43:46,317] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:43:46,387] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:43:46,497] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:43:46,503] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:43:46,507] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:43:46,509] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:43:46,510] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:43:46,513] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:43:46,516] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:43:46,519] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:43:46,520] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:43:46,523] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:43:46,525] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:43:46,528] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:43:46,529] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:43:46,532] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:43:46,534] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:43:46,537] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:43:52,401] INFO : VALIDATION | Step=32000/50000 | Train Loss=8.4876 | Valid Loss=8.3453 | Train Loss Unweighted=2.9761 | Valid Loss Unweighted=3.0147 | Stale=0 | ETA=51.4m +rank0 [2026-10-28 12:43:52,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:43:52,467] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:43:52,470] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:43:52,643] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:43:54,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:43:54,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:43:55,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:43:56,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:43:56,361] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:43:56,364] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:43:56,420] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:43:56,423] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:43:56,596] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:43:58,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:43:58,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:43:59,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:43:59,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:43:59,939] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:43:59,943] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:43:59,997] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:44:00,000] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:44:00,122] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:44:01,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:44:02,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:44:02,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:44:03,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:44:03,327] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:44:03,335] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:44:03,593] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:44:04,001] INFO : Plotted number of elements +rank0 [2026-10-28 12:44:05,474] INFO : Plotted jets +rank0 [2026-10-28 12:44:06,555] INFO : Plotted jet ratio +rank0 [2026-10-28 12:44:06,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:44:06,820] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:44:07,213] INFO : Plotted number of elements +rank0 [2026-10-28 12:44:08,611] INFO : Plotted jets +rank0 [2026-10-28 12:44:10,259] INFO : Plotted jet ratio +rank0 [2026-10-28 12:44:10,339] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 12:44:10,543] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 12:44:10,912] INFO : Plotted number of elements +rank0 [2026-10-28 12:44:12,301] INFO : Plotted jets +rank0 [2026-10-28 12:44:13,358] INFO : Plotted jet ratio +rank0 [2026-10-28 12:44:24,706] INFO : Step 32100/50000 rank0 | Train Loss: 8.5814 | Train Loss Unweighted: 3.0028 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6382 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1625 | Regression_eta: 0.0115 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7590 | Classification_binary: 2.6382 | Regression_cos_phi: 0.4249 | Regression_energy: 1.7769 | Regression_eta: 0.5131 | Regression_pt: 1.8724 | Regression_sin_phi: 0.5969 | LR: 3.56e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.4708s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0165 | max_loss: 0.0624 | max_backward: 0.0383 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3868 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5215.4 | uss_MB: 4747.7 | child_rss_MB: 3196.7 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:44:24,921] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:35,775] INFO : Step 32200/50000 rank0 | Train Loss: 8.9501 | Train Loss Unweighted: 3.1769 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8049 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1668 | Regression_eta: 0.0113 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7476 | Classification_binary: 2.8049 | Regression_cos_phi: 0.5090 | Regression_energy: 1.8234 | Regression_eta: 0.5054 | Regression_pt: 1.8906 | Regression_sin_phi: 0.6692 | LR: 3.53e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4158s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0547 | max_backward: 0.0338 | max_optimizer_step: 0.1275 | max_tensorboard_logging: 0.3808 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5096.2 | uss_MB: 4627.3 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:44:35,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:47,003] INFO : Step 32300/50000 rank0 | Train Loss: 8.2214 | Train Loss Unweighted: 2.9704 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6436 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1429 | Regression_eta: 0.0097 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7505 | Classification_binary: 2.6436 | Regression_cos_phi: 0.5937 | Regression_energy: 1.5628 | Regression_eta: 0.4338 | Regression_pt: 1.6499 | Regression_sin_phi: 0.5871 | LR: 3.51e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4173s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0671 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.3934 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 5097.2 | uss_MB: 4628.0 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:44:47,218] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:58,134] INFO : Step 32400/50000 rank0 | Train Loss: 8.3067 | Train Loss Unweighted: 2.9954 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6556 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1525 | Regression_eta: 0.0102 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6935 | Classification_binary: 2.6556 | Regression_cos_phi: 0.5204 | Regression_energy: 1.6668 | Regression_eta: 0.4566 | Regression_pt: 1.7145 | Regression_sin_phi: 0.5994 | LR: 3.48e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4073s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0476 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.4008 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5097.2 | uss_MB: 4628.1 | child_rss_MB: 3198.0 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:44:58,346] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:09,205] INFO : Step 32500/50000 rank0 | Train Loss: 8.3416 | Train Loss Unweighted: 2.9858 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.6567 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1447 | Regression_eta: 0.0131 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6621 | Classification_binary: 2.6567 | Regression_cos_phi: 0.5032 | Regression_energy: 1.5823 | Regression_eta: 0.5847 | Regression_pt: 1.6453 | Regression_sin_phi: 0.7073 | LR: 3.46e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3479s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0302 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1248 | max_tensorboard_logging: 0.3733 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5097.2 | uss_MB: 4628.1 | child_rss_MB: 3200.1 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:45:09,417] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:20,312] INFO : Step 32600/50000 rank0 | Train Loss: 8.8301 | Train Loss Unweighted: 3.1125 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7620 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1541 | Regression_eta: 0.0120 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8028 | Classification_binary: 2.7620 | Regression_cos_phi: 0.6391 | Regression_energy: 1.6843 | Regression_eta: 0.5349 | Regression_pt: 1.7374 | Regression_sin_phi: 0.6696 | LR: 3.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4426s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1110 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3710 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5097.2 | uss_MB: 4628.2 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:45:20,526] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:31,426] INFO : Step 32700/50000 rank0 | Train Loss: 8.4444 | Train Loss Unweighted: 3.0079 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6729 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1455 | Regression_eta: 0.0137 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7941 | Classification_binary: 2.6729 | Regression_cos_phi: 0.4349 | Regression_energy: 1.5902 | Regression_eta: 0.6118 | Regression_pt: 1.6749 | Regression_sin_phi: 0.6656 | LR: 3.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3669s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1051 | max_forward: 0.0136 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.2851 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5097.2 | uss_MB: 4628.5 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:45:31,639] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:42,555] INFO : Step 32800/50000 rank0 | Train Loss: 8.5610 | Train Loss Unweighted: 3.0593 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7064 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1555 | Regression_eta: 0.0126 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7337 | Classification_binary: 2.7064 | Regression_cos_phi: 0.4497 | Regression_energy: 1.7001 | Regression_eta: 0.5616 | Regression_pt: 1.8060 | Regression_sin_phi: 0.6035 | LR: 3.38e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3394s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0504 | max_forward: 0.0137 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5096.2 | uss_MB: 4627.6 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:45:42,769] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:53,506] INFO : Step 32900/50000 rank0 | Train Loss: 9.1049 | Train Loss Unweighted: 3.0897 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.7089 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1670 | Regression_eta: 0.0136 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7688 | Classification_binary: 2.7089 | Regression_cos_phi: 0.4743 | Regression_energy: 1.8259 | Regression_eta: 0.6073 | Regression_pt: 1.9533 | Regression_sin_phi: 0.7663 | LR: 3.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0797 | max_backward: 0.0332 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5096.2 | uss_MB: 4627.6 | child_rss_MB: 3199.6 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:45:53,718] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:04,750] INFO : Step 33000/50000 rank0 | Train Loss: 7.8995 | Train Loss Unweighted: 2.9063 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5945 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1365 | Regression_eta: 0.0128 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6756 | Classification_binary: 2.5945 | Regression_cos_phi: 0.3831 | Regression_energy: 1.4925 | Regression_eta: 0.5690 | Regression_pt: 1.5710 | Regression_sin_phi: 0.6138 | LR: 3.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3854s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0327 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3802 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5096.2 | uss_MB: 4630.2 | child_rss_MB: 3200.5 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:46:04,961] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:04,989] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 12:46:05,072] INFO : Running validation on rank0 +rank0 [2026-10-28 12:46:05,073] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:46:05,073] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:46:06,173] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 12:46:06,320] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:46:06,453] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:46:06,528] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:46:06,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:46:06,666] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:46:06,736] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:46:06,806] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:46:06,915] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:46:07,039] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:46:07,161] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:46:07,233] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:46:07,303] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:46:07,372] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:46:07,441] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:46:07,510] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:46:07,619] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:46:07,626] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:46:07,630] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:46:07,634] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:46:07,638] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:46:07,642] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:46:07,646] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:46:07,650] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:46:07,653] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:46:07,655] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:46:07,658] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:46:07,662] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:46:07,663] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:46:07,667] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:46:07,671] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:46:07,675] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:46:13,588] INFO : VALIDATION | Step=33000/50000 | Train Loss=7.8995 | Valid Loss=8.3631 | Train Loss Unweighted=2.9063 | Valid Loss Unweighted=3.0176 | Stale=1 | ETA=48.3m +rank0 [2026-10-28 12:46:13,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:13,657] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:46:13,660] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:46:13,829] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:15,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:16,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:16,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:17,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:17,519] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:46:17,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:17,578] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:46:17,581] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:17,704] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:19,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:19,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:20,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:20,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:21,117] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:46:21,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:21,174] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:46:21,177] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:46:21,290] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:22,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:23,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:23,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:24,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:24,564] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:46:24,572] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:46:24,836] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:46:25,242] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:26,731] INFO : Plotted jets +rank0 [2026-10-28 12:46:27,829] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:27,862] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:28,095] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:28,495] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:30,434] INFO : Plotted jets +rank0 [2026-10-28 12:46:31,538] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:31,571] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 12:46:31,782] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 12:46:32,144] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:33,523] INFO : Plotted jets +rank0 [2026-10-28 12:46:34,587] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:46,050] INFO : Step 33100/50000 rank0 | Train Loss: 8.4348 | Train Loss Unweighted: 3.0396 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7067 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1457 | Regression_eta: 0.0148 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7155 | Classification_binary: 2.7067 | Regression_cos_phi: 0.4622 | Regression_energy: 1.5931 | Regression_eta: 0.6597 | Regression_pt: 1.6573 | Regression_sin_phi: 0.6402 | LR: 3.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4993s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1195 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1256 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5293.5 | uss_MB: 4825.3 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:46:46,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:57,133] INFO : Step 33200/50000 rank0 | Train Loss: 7.9641 | Train Loss Unweighted: 2.9584 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6445 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1368 | Regression_eta: 0.0103 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.6445 | Regression_cos_phi: 0.4414 | Regression_energy: 1.4956 | Regression_eta: 0.4598 | Regression_pt: 1.6082 | Regression_sin_phi: 0.6446 | LR: 3.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3999s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0439 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1273 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5169.8 | uss_MB: 4700.3 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:46:57,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:08,251] INFO : Step 33300/50000 rank0 | Train Loss: 7.9828 | Train Loss Unweighted: 3.0623 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.7486 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1383 | Regression_eta: 0.0086 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7189 | Classification_binary: 2.7486 | Regression_cos_phi: 0.4388 | Regression_energy: 1.5115 | Regression_eta: 0.3852 | Regression_pt: 1.6026 | Regression_sin_phi: 0.5773 | LR: 3.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4362s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0514 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1261 | max_tensorboard_logging: 0.3926 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5169.8 | uss_MB: 4700.5 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:47:08,465] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:19,305] INFO : Step 33400/50000 rank0 | Train Loss: 8.7768 | Train Loss Unweighted: 3.0816 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7286 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1550 | Regression_eta: 0.0098 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7943 | Classification_binary: 2.7286 | Regression_cos_phi: 0.4634 | Regression_energy: 1.6942 | Regression_eta: 0.4358 | Regression_pt: 1.7930 | Regression_sin_phi: 0.8676 | LR: 3.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3387s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3764 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5170.8 | uss_MB: 4700.8 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:19,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:30,340] INFO : Step 33500/50000 rank0 | Train Loss: 8.5685 | Train Loss Unweighted: 2.9367 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5946 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1482 | Regression_eta: 0.0162 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6914 | Classification_binary: 2.5946 | Regression_cos_phi: 0.5011 | Regression_energy: 1.6199 | Regression_eta: 0.7219 | Regression_pt: 1.7110 | Regression_sin_phi: 0.7287 | LR: 3.21e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4096s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0333 | max_backward: 0.0334 | max_optimizer_step: 0.1262 | max_tensorboard_logging: 0.3922 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5170.8 | uss_MB: 4700.9 | child_rss_MB: 3196.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:30,552] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:41,573] INFO : Step 33600/50000 rank0 | Train Loss: 8.8185 | Train Loss Unweighted: 3.0817 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7129 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1654 | Regression_eta: 0.0139 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7447 | Classification_binary: 2.7129 | Regression_cos_phi: 0.4824 | Regression_energy: 1.8083 | Regression_eta: 0.6185 | Regression_pt: 1.8524 | Regression_sin_phi: 0.5993 | LR: 3.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4486s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0663 | max_loss: 0.0033 | max_backward: 0.0343 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.3990 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 5170.8 | uss_MB: 4701.1 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:47:41,785] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:52,633] INFO : Step 33700/50000 rank0 | Train Loss: 8.2483 | Train Loss Unweighted: 3.0850 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.7466 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1503 | Regression_eta: 0.0105 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6459 | Classification_binary: 2.7466 | Regression_cos_phi: 0.3985 | Regression_energy: 1.6433 | Regression_eta: 0.4682 | Regression_pt: 1.7596 | Regression_sin_phi: 0.5862 | LR: 3.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4612s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1091 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5171.8 | uss_MB: 4701.3 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:52,848] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:03,736] INFO : Step 33800/50000 rank0 | Train Loss: 8.5199 | Train Loss Unweighted: 3.0325 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6685 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1628 | Regression_eta: 0.0100 | Regression_pt: 0.1589 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7425 | Classification_binary: 2.6685 | Regression_cos_phi: 0.4161 | Regression_energy: 1.7800 | Regression_eta: 0.4446 | Regression_pt: 1.8885 | Regression_sin_phi: 0.5797 | LR: 3.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3961s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0333 | max_loss: 0.0036 | max_backward: 0.0349 | max_optimizer_step: 0.1294 | max_tensorboard_logging: 0.3627 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5170.5 | uss_MB: 4700.4 | child_rss_MB: 3201.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:48:03,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:14,824] INFO : Step 33900/50000 rank0 | Train Loss: 8.6038 | Train Loss Unweighted: 2.9823 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6427 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1476 | Regression_eta: 0.0188 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6926 | Classification_binary: 2.6427 | Regression_cos_phi: 0.4657 | Regression_energy: 1.6132 | Regression_eta: 0.8364 | Regression_pt: 1.6701 | Regression_sin_phi: 0.6831 | LR: 3.11e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0584 | max_backward: 0.0340 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.3969 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5170.5 | uss_MB: 4700.3 | child_rss_MB: 3201.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:48:15,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:26,143] INFO : Step 34000/50000 rank0 | Train Loss: 8.7485 | Train Loss Unweighted: 2.9757 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.6261 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1533 | Regression_eta: 0.0149 | Regression_pt: 0.1441 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7908 | Classification_binary: 2.6261 | Regression_cos_phi: 0.5675 | Regression_energy: 1.6765 | Regression_eta: 0.6667 | Regression_pt: 1.7127 | Regression_sin_phi: 0.7084 | LR: 3.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4657s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1141 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5170.5 | uss_MB: 4700.8 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:48:26,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:26,390] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 12:48:26,473] INFO : Running validation on rank0 +rank0 [2026-10-28 12:48:26,473] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:48:26,473] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:48:27,572] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 12:48:27,718] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:48:27,851] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:48:27,925] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:48:27,996] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:48:28,064] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:48:28,134] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:48:28,204] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:48:28,312] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:48:28,437] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:48:28,560] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:48:28,633] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:48:28,703] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:48:28,772] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:48:28,842] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:48:28,912] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:48:29,022] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:48:29,029] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:48:29,032] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:48:29,034] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:48:29,035] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:48:29,038] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:48:29,041] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:48:29,044] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:48:29,046] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:48:29,049] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:48:29,052] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:48:29,054] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:48:29,055] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:48:29,058] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:48:29,060] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:48:29,061] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:48:35,141] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.7485 | Valid Loss=8.2430 | Train Loss Unweighted=2.9757 | Valid Loss Unweighted=2.9711 | Stale=0 | ETA=45.2m +rank0 [2026-10-28 12:48:35,142] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:48:35,199] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:48:35,204] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:48:35,379] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:48:36,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:48:37,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:48:37,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:48:38,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:48:38,690] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:48:38,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:48:38,748] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:48:38,751] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:48:38,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:48:40,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:48:41,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:48:41,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:48:42,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:48:42,341] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:48:42,345] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:48:42,399] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:48:42,402] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:48:42,516] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:48:44,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:48:44,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:48:45,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:48:45,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:48:45,732] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:48:45,739] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:48:46,006] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:48:46,416] INFO : Plotted number of elements +rank0 [2026-10-28 12:48:47,867] INFO : Plotted jets +rank0 [2026-10-28 12:48:48,956] INFO : Plotted jet ratio +rank0 [2026-10-28 12:48:48,990] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:48:49,219] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:48:49,614] INFO : Plotted number of elements +rank0 [2026-10-28 12:48:51,007] INFO : Plotted jets +rank0 [2026-10-28 12:48:52,623] INFO : Plotted jet ratio +rank0 [2026-10-28 12:48:52,654] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 12:48:52,861] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 12:48:53,230] INFO : Plotted number of elements +rank0 [2026-10-28 12:48:54,616] INFO : Plotted jets +rank0 [2026-10-28 12:48:55,738] INFO : Plotted jet ratio +rank0 [2026-10-28 12:49:07,151] INFO : Step 34100/50000 rank0 | Train Loss: 8.3385 | Train Loss Unweighted: 3.0091 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.6780 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1452 | Regression_eta: 0.0118 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7389 | Classification_binary: 2.6780 | Regression_cos_phi: 0.4757 | Regression_energy: 1.5875 | Regression_eta: 0.5265 | Regression_pt: 1.6654 | Regression_sin_phi: 0.6667 | LR: 3.06e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4008s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0223 | max_loss: 0.0033 | max_backward: 0.0348 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.3931 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5363.9 | uss_MB: 4894.4 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:49:07,364] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:18,258] INFO : Step 34200/50000 rank0 | Train Loss: 8.8190 | Train Loss Unweighted: 3.1324 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7574 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1669 | Regression_eta: 0.0088 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6972 | Classification_binary: 2.7574 | Regression_cos_phi: 0.4431 | Regression_energy: 1.8252 | Regression_eta: 0.3927 | Regression_pt: 1.9756 | Regression_sin_phi: 0.7279 | LR: 3.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4084s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1068 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3142 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5241.8 | uss_MB: 4773.3 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:18,474] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:29,358] INFO : Step 34300/50000 rank0 | Train Loss: 8.1188 | Train Loss Unweighted: 2.9349 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6307 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1300 | Regression_eta: 0.0138 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7211 | Classification_binary: 2.6307 | Regression_cos_phi: 0.5174 | Regression_energy: 1.4209 | Regression_eta: 0.6175 | Regression_pt: 1.4933 | Regression_sin_phi: 0.7180 | LR: 3.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3908s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0897 | max_forward: 0.0180 | max_loss: 0.0035 | max_backward: 0.0393 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.2953 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5242.8 | uss_MB: 4776.4 | child_rss_MB: 3195.2 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:49:29,570] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:40,623] INFO : Step 34400/50000 rank0 | Train Loss: 8.6032 | Train Loss Unweighted: 3.0206 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6695 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1554 | Regression_eta: 0.0142 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6789 | Classification_binary: 2.6695 | Regression_cos_phi: 0.4902 | Regression_energy: 1.6984 | Regression_eta: 0.6347 | Regression_pt: 1.7700 | Regression_sin_phi: 0.6615 | LR: 2.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3468s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.2688 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5242.8 | uss_MB: 4774.9 | child_rss_MB: 3192.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:40,847] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:51,724] INFO : Step 34500/50000 rank0 | Train Loss: 7.5407 | Train Loss Unweighted: 2.9423 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.6333 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1386 | Regression_eta: 0.0077 | Regression_pt: 0.1352 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6682 | Classification_binary: 2.6333 | Regression_cos_phi: 0.3403 | Regression_energy: 1.5153 | Regression_eta: 0.3436 | Regression_pt: 1.6062 | Regression_sin_phi: 0.4339 | LR: 2.97e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3368s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0130 | Memory: rss_MB: 5242.8 | uss_MB: 4775.4 | child_rss_MB: 3194.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:51,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:02,834] INFO : Step 34600/50000 rank0 | Train Loss: 8.7762 | Train Loss Unweighted: 3.0292 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6894 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1475 | Regression_eta: 0.0174 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8151 | Classification_binary: 2.6894 | Regression_cos_phi: 0.4474 | Regression_energy: 1.6129 | Regression_eta: 0.7770 | Regression_pt: 1.6380 | Regression_sin_phi: 0.7964 | LR: 2.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.2958 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5242.8 | uss_MB: 4775.4 | child_rss_MB: 3193.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:50:03,049] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:13,670] INFO : Step 34700/50000 rank0 | Train Loss: 8.1281 | Train Loss Unweighted: 3.0442 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7223 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1411 | Regression_eta: 0.0077 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6950 | Classification_binary: 2.7223 | Regression_cos_phi: 0.5339 | Regression_energy: 1.5429 | Regression_eta: 0.3425 | Regression_pt: 1.6609 | Regression_sin_phi: 0.6306 | LR: 2.92e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3106s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0571 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.2774 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5242.8 | uss_MB: 4775.4 | child_rss_MB: 3193.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:50:13,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:24,791] INFO : Step 34800/50000 rank0 | Train Loss: 8.1722 | Train Loss Unweighted: 2.8989 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5651 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1449 | Regression_eta: 0.0092 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7600 | Classification_binary: 2.5651 | Regression_cos_phi: 0.5810 | Regression_energy: 1.5837 | Regression_eta: 0.4100 | Regression_pt: 1.7197 | Regression_sin_phi: 0.5525 | LR: 2.90e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3630s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0725 | max_forward: 0.0173 | max_loss: 0.0035 | max_backward: 0.0382 | max_optimizer_step: 0.1275 | max_tensorboard_logging: 0.2998 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5241.1 | uss_MB: 4775.5 | child_rss_MB: 3201.2 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 12:50:25,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:35,912] INFO : Step 34900/50000 rank0 | Train Loss: 9.0481 | Train Loss Unweighted: 3.0615 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.6655 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1771 | Regression_eta: 0.0127 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7018 | Classification_binary: 2.6655 | Regression_cos_phi: 0.4905 | Regression_energy: 1.9363 | Regression_eta: 0.5675 | Regression_pt: 2.0595 | Regression_sin_phi: 0.6271 | LR: 2.88e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4051s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0749 | max_forward: 0.0135 | max_loss: 0.0035 | max_backward: 0.0330 | max_optimizer_step: 0.1268 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 5241.1 | uss_MB: 4774.4 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:50:36,128] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:46,972] INFO : Step 35000/50000 rank0 | Train Loss: 8.2289 | Train Loss Unweighted: 2.9858 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.6656 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1404 | Regression_eta: 0.0110 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7033 | Classification_binary: 2.6656 | Regression_cos_phi: 0.5227 | Regression_energy: 1.5351 | Regression_eta: 0.4909 | Regression_pt: 1.5977 | Regression_sin_phi: 0.7136 | LR: 2.85e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4003s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1015 | max_forward: 0.0170 | max_loss: 0.0033 | max_backward: 0.0389 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.2963 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5242.1 | uss_MB: 4777.1 | child_rss_MB: 3198.7 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:50:47,188] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:47,206] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 12:50:47,290] INFO : Running validation on rank0 +rank0 [2026-10-28 12:50:47,290] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:50:47,291] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:50:48,397] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 12:50:48,546] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:50:48,680] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:50:48,754] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:50:48,823] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:50:48,891] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:50:48,961] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:50:49,030] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:50:49,140] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:50:49,273] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:50:49,406] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:50:49,477] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:50:49,547] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:50:49,616] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:50:49,687] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:50:49,757] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:50:49,868] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:50:49,874] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:50:49,877] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:50:49,880] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:50:49,883] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:50:49,884] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:50:49,888] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:50:49,890] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:50:49,891] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:50:49,894] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:50:50,000] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:50:50,003] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:50:50,005] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:50:50,006] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:50:50,009] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:50:50,012] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:50:55,957] INFO : VALIDATION | Step=35000/50000 | Train Loss=8.2289 | Valid Loss=8.3163 | Train Loss Unweighted=2.9858 | Valid Loss Unweighted=2.9814 | Stale=1 | ETA=42.2m +rank0 [2026-10-28 12:50:55,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:50:56,021] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:50:56,025] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:50:56,195] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:50:57,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:50:58,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:50:58,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:50:59,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:50:59,732] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:50:59,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:50:59,789] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:50:59,792] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:50:59,904] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:51:01,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:51:01,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:51:02,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:51:03,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:51:03,231] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:51:03,234] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:51:03,287] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:51:03,290] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:51:03,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:51:04,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:51:05,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:51:05,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:51:06,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:51:06,735] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:51:06,743] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:51:07,004] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:51:07,410] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:08,859] INFO : Plotted jets +rank0 [2026-10-28 12:51:09,958] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:09,991] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:51:10,226] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:51:10,622] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:12,015] INFO : Plotted jets +rank0 [2026-10-28 12:51:13,670] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:13,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 12:51:13,913] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 12:51:14,279] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:15,661] INFO : Plotted jets +rank0 [2026-10-28 12:51:16,725] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:28,045] INFO : Step 35100/50000 rank0 | Train Loss: 8.3284 | Train Loss Unweighted: 2.8907 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5623 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1406 | Regression_eta: 0.0153 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7815 | Classification_binary: 2.5623 | Regression_cos_phi: 0.5260 | Regression_energy: 1.5370 | Regression_eta: 0.6844 | Regression_pt: 1.6306 | Regression_sin_phi: 0.6065 | LR: 2.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4442s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0159 | max_loss: 0.0545 | max_backward: 0.0382 | max_optimizer_step: 0.1275 | max_tensorboard_logging: 0.4023 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 5428.8 | uss_MB: 4962.6 | child_rss_MB: 3197.8 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:51:28,259] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:39,165] INFO : Step 35200/50000 rank0 | Train Loss: 7.7358 | Train Loss Unweighted: 2.9265 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.6100 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1426 | Regression_eta: 0.0099 | Regression_pt: 0.1358 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6572 | Classification_binary: 2.6100 | Regression_cos_phi: 0.3594 | Regression_energy: 1.5589 | Regression_eta: 0.4408 | Regression_pt: 1.6138 | Regression_sin_phi: 0.4958 | LR: 2.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4828s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0702 | max_backward: 0.0332 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.4188 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5317.7 | uss_MB: 4848.5 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:51:39,378] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:50,373] INFO : Step 35300/50000 rank0 | Train Loss: 8.1360 | Train Loss Unweighted: 2.9387 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6074 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1460 | Regression_eta: 0.0097 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6776 | Classification_binary: 2.6074 | Regression_cos_phi: 0.4708 | Regression_energy: 1.5965 | Regression_eta: 0.4308 | Regression_pt: 1.7055 | Regression_sin_phi: 0.6475 | LR: 2.79e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.4164s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0243 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3992 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5318.7 | uss_MB: 4849.6 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:51:50,587] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:01,439] INFO : Step 35400/50000 rank0 | Train Loss: 7.8836 | Train Loss Unweighted: 2.9146 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6095 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1329 | Regression_eta: 0.0092 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7844 | Classification_binary: 2.6095 | Regression_cos_phi: 0.4326 | Regression_energy: 1.4529 | Regression_eta: 0.4090 | Regression_pt: 1.5256 | Regression_sin_phi: 0.6696 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4874s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1118 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1297 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5318.7 | uss_MB: 4849.8 | child_rss_MB: 3194.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:52:01,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:12,518] INFO : Step 35500/50000 rank0 | Train Loss: 8.5601 | Train Loss Unweighted: 3.0585 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7013 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1618 | Regression_eta: 0.0103 | Regression_pt: 0.1522 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7156 | Classification_binary: 2.7013 | Regression_cos_phi: 0.4536 | Regression_energy: 1.7694 | Regression_eta: 0.4588 | Regression_pt: 1.8086 | Regression_sin_phi: 0.6529 | LR: 2.74e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4075s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0633 | max_loss: 0.0034 | max_backward: 0.0388 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3705 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5319.7 | uss_MB: 4852.3 | child_rss_MB: 3193.4 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:52:12,730] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:23,658] INFO : Step 35600/50000 rank0 | Train Loss: 8.4640 | Train Loss Unweighted: 2.9692 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6124 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1582 | Regression_eta: 0.0113 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7311 | Classification_binary: 2.6124 | Regression_cos_phi: 0.3885 | Regression_energy: 1.7290 | Regression_eta: 0.5020 | Regression_pt: 1.8417 | Regression_sin_phi: 0.6592 | LR: 2.72e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4066s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0369 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.3834 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5319.7 | uss_MB: 4850.2 | child_rss_MB: 3196.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:52:23,869] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:34,647] INFO : Step 35700/50000 rank0 | Train Loss: 8.2471 | Train Loss Unweighted: 3.0628 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7434 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1405 | Regression_eta: 0.0076 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7120 | Classification_binary: 2.7434 | Regression_cos_phi: 0.5157 | Regression_energy: 1.5357 | Regression_eta: 0.3375 | Regression_pt: 1.6186 | Regression_sin_phi: 0.7842 | LR: 2.70e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4826s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0133 | max_loss: 0.0827 | max_backward: 0.0330 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5319.7 | uss_MB: 4850.2 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:52:34,857] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:45,851] INFO : Step 35800/50000 rank0 | Train Loss: 8.2720 | Train Loss Unweighted: 2.9460 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5977 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1560 | Regression_eta: 0.0089 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7133 | Classification_binary: 2.5977 | Regression_cos_phi: 0.4603 | Regression_energy: 1.7059 | Regression_eta: 0.3981 | Regression_pt: 1.7932 | Regression_sin_phi: 0.6034 | LR: 2.68e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4909s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1231 | max_loss: 0.0035 | max_backward: 0.0387 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3771 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 5319.0 | uss_MB: 4850.7 | child_rss_MB: 3197.7 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:52:46,065] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:56,806] INFO : Step 35900/50000 rank0 | Train Loss: 7.9205 | Train Loss Unweighted: 2.9577 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6349 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1433 | Regression_eta: 0.0077 | Regression_pt: 0.1401 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7573 | Classification_binary: 2.6349 | Regression_cos_phi: 0.3218 | Regression_energy: 1.5663 | Regression_eta: 0.3446 | Regression_pt: 1.6643 | Regression_sin_phi: 0.6313 | LR: 2.65e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4117s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0628 | max_backward: 0.0332 | max_optimizer_step: 0.1280 | max_tensorboard_logging: 0.3707 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5319.0 | uss_MB: 4849.5 | child_rss_MB: 3195.2 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:52:57,019] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:07,702] INFO : Step 36000/50000 rank0 | Train Loss: 9.1170 | Train Loss Unweighted: 3.1454 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7841 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1610 | Regression_eta: 0.0228 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7169 | Classification_binary: 2.7841 | Regression_cos_phi: 0.5516 | Regression_energy: 1.7597 | Regression_eta: 1.0162 | Regression_pt: 1.7099 | Regression_sin_phi: 0.5787 | LR: 2.63e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2859s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0673 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5319.0 | uss_MB: 4849.7 | child_rss_MB: 3220.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:53:07,917] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:07,937] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 12:53:08,021] INFO : Running validation on rank0 +rank0 [2026-10-28 12:53:08,021] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:53:08,021] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:53:09,210] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 12:53:09,355] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:53:09,487] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:53:09,562] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:53:09,631] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:53:09,700] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:53:09,771] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:53:09,842] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:53:09,953] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:53:10,087] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:53:10,219] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:53:10,291] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:53:10,361] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:53:10,430] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:53:10,502] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:53:10,573] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:53:10,682] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:53:10,689] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:53:10,691] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:53:10,692] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:53:10,696] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:53:10,698] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:53:10,699] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:53:10,702] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:53:10,705] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:53:10,708] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:53:10,709] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:53:10,711] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:53:10,712] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:53:10,714] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:53:10,717] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:53:10,718] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:53:16,578] INFO : VALIDATION | Step=36000/50000 | Train Loss=9.1170 | Valid Loss=8.2770 | Train Loss Unweighted=3.1454 | Valid Loss Unweighted=2.9803 | Stale=2 | ETA=39.2m +rank0 [2026-10-28 12:53:16,586] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:16,644] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:53:16,648] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:53:16,868] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:18,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:19,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:19,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:20,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:20,514] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:53:20,518] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:20,573] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:53:20,576] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:20,693] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:22,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:22,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:23,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:23,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:23,957] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:53:23,960] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:24,013] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:53:24,016] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:53:24,131] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:25,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:26,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:26,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:27,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:27,582] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:53:27,590] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:53:27,848] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:53:28,267] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:29,706] INFO : Plotted jets +rank0 [2026-10-28 12:53:30,800] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:30,833] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:31,066] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:31,460] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:32,858] INFO : Plotted jets +rank0 [2026-10-28 12:53:34,469] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:34,502] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 12:53:34,712] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 12:53:35,077] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:36,453] INFO : Plotted jets +rank0 [2026-10-28 12:53:37,512] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:48,856] INFO : Step 36100/50000 rank0 | Train Loss: 7.8103 | Train Loss Unweighted: 2.9499 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6354 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1406 | Regression_eta: 0.0098 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7290 | Classification_binary: 2.6354 | Regression_cos_phi: 0.3428 | Regression_energy: 1.5367 | Regression_eta: 0.4380 | Regression_pt: 1.5889 | Regression_sin_phi: 0.5396 | LR: 2.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4449s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1317 | max_tensorboard_logging: 0.3981 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5513.3 | uss_MB: 5043.9 | child_rss_MB: 3193.9 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:49,074] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:53:59,952] INFO : Step 36200/50000 rank0 | Train Loss: 7.8587 | Train Loss Unweighted: 2.8574 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5409 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1388 | Regression_eta: 0.0077 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7148 | Classification_binary: 2.5409 | Regression_cos_phi: 0.3797 | Regression_energy: 1.5169 | Regression_eta: 0.3428 | Regression_pt: 1.6335 | Regression_sin_phi: 0.7300 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4947s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0134 | max_loss: 0.0752 | max_backward: 0.0328 | max_optimizer_step: 0.1285 | max_tensorboard_logging: 0.3954 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 5513.3 | uss_MB: 5043.9 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:54:00,165] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:11,146] INFO : Step 36300/50000 rank0 | Train Loss: 8.5631 | Train Loss Unweighted: 3.0412 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6711 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1677 | Regression_eta: 0.0078 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6914 | Classification_binary: 2.6711 | Regression_cos_phi: 0.4340 | Regression_energy: 1.8336 | Regression_eta: 0.3488 | Regression_pt: 1.9329 | Regression_sin_phi: 0.6515 | LR: 2.57e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4081s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0234 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3998 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 5394.7 | uss_MB: 4924.8 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:54:11,364] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:22,202] INFO : Step 36400/50000 rank0 | Train Loss: 7.5979 | Train Loss Unweighted: 2.8414 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5524 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1250 | Regression_eta: 0.0090 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7531 | Classification_binary: 2.5524 | Regression_cos_phi: 0.4524 | Regression_energy: 1.3670 | Regression_eta: 0.3999 | Regression_pt: 1.4422 | Regression_sin_phi: 0.6308 | LR: 2.55e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3442s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0844 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.2863 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5395.7 | uss_MB: 4925.0 | child_rss_MB: 3194.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:54:22,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:33,303] INFO : Step 36500/50000 rank0 | Train Loss: 8.3497 | Train Loss Unweighted: 2.9112 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5655 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1532 | Regression_eta: 0.0112 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7607 | Classification_binary: 2.5655 | Regression_cos_phi: 0.4211 | Regression_energy: 1.6752 | Regression_eta: 0.4994 | Regression_pt: 1.7516 | Regression_sin_phi: 0.6762 | LR: 2.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3811s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0173 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5396.7 | uss_MB: 4927.6 | child_rss_MB: 3192.4 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:54:33,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:44,408] INFO : Step 36600/50000 rank0 | Train Loss: 9.0300 | Train Loss Unweighted: 3.1079 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7445 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1602 | Regression_eta: 0.0155 | Regression_pt: 0.1513 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7455 | Classification_binary: 2.7445 | Regression_cos_phi: 0.5862 | Regression_energy: 1.7514 | Regression_eta: 0.6904 | Regression_pt: 1.7983 | Regression_sin_phi: 0.7136 | LR: 2.50e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4443s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0845 | max_loss: 0.0035 | max_backward: 0.0341 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3619 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5396.7 | uss_MB: 4925.5 | child_rss_MB: 3192.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:54:44,620] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:55,517] INFO : Step 36700/50000 rank0 | Train Loss: 8.8857 | Train Loss Unweighted: 3.1248 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7694 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1568 | Regression_eta: 0.0161 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7276 | Classification_binary: 2.7694 | Regression_cos_phi: 0.4736 | Regression_energy: 1.7147 | Regression_eta: 0.7184 | Regression_pt: 1.7618 | Regression_sin_phi: 0.7203 | LR: 2.48e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0605 | max_loss: 0.0036 | max_backward: 0.0333 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5396.7 | uss_MB: 4925.6 | child_rss_MB: 3192.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:54:55,729] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:06,630] INFO : Step 36800/50000 rank0 | Train Loss: 8.4058 | Train Loss Unweighted: 3.0876 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7543 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1468 | Regression_eta: 0.0096 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7157 | Classification_binary: 2.7543 | Regression_cos_phi: 0.5575 | Regression_energy: 1.6054 | Regression_eta: 0.4272 | Regression_pt: 1.6904 | Regression_sin_phi: 0.6553 | LR: 2.46e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3838s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0887 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0384 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5396.7 | uss_MB: 4927.6 | child_rss_MB: 3194.8 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:55:06,846] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:17,705] INFO : Step 36900/50000 rank0 | Train Loss: 8.2571 | Train Loss Unweighted: 2.9882 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6650 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1408 | Regression_eta: 0.0123 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7247 | Classification_binary: 2.6650 | Regression_cos_phi: 0.4916 | Regression_energy: 1.5395 | Regression_eta: 0.5486 | Regression_pt: 1.6182 | Regression_sin_phi: 0.6694 | LR: 2.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3913s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1021 | max_forward: 0.0155 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.2894 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5395.1 | uss_MB: 4924.9 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:55:17,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:28,785] INFO : Step 37000/50000 rank0 | Train Loss: 8.5764 | Train Loss Unweighted: 2.9920 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6459 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1530 | Regression_eta: 0.0139 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6730 | Classification_binary: 2.6459 | Regression_cos_phi: 0.4610 | Regression_energy: 1.6731 | Regression_eta: 0.6216 | Regression_pt: 1.7346 | Regression_sin_phi: 0.7671 | LR: 2.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0201 | max_forward: 0.0168 | max_loss: 0.0035 | max_backward: 0.0383 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5396.1 | uss_MB: 4927.4 | child_rss_MB: 3196.1 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:55:28,996] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:29,013] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 12:55:29,097] INFO : Running validation on rank0 +rank0 [2026-10-28 12:55:29,097] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:55:29,097] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:55:30,187] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 12:55:30,335] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:55:30,469] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:55:30,543] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:55:30,612] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:55:30,680] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:55:30,750] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:55:30,820] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:55:30,931] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:55:31,064] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:55:31,197] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:55:31,270] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:55:31,339] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:55:31,408] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:55:31,477] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:55:31,548] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:55:31,660] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:55:31,667] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:55:31,670] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:55:31,673] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:55:31,673] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:55:31,674] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:55:31,677] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:55:31,680] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:55:31,682] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:55:31,684] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:55:31,686] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:55:31,687] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:55:31,690] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:55:31,692] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:55:31,694] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:55:31,696] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:55:37,569] INFO : VALIDATION | Step=37000/50000 | Train Loss=8.5764 | Valid Loss=8.1542 | Train Loss Unweighted=2.9920 | Valid Loss Unweighted=2.9461 | Stale=0 | ETA=36.2m +rank0 [2026-10-28 12:55:37,578] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:55:37,635] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:55:37,639] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:55:37,811] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:55:39,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:55:40,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:55:40,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:55:41,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:55:41,509] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:55:41,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:55:41,566] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:55:41,577] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:55:41,757] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:55:43,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:55:43,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:55:44,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:55:45,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:55:45,181] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:55:45,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:55:45,239] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:55:45,242] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:55:45,352] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:55:46,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:55:47,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:55:47,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:55:48,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:55:48,641] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:55:48,649] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:55:48,911] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:55:49,318] INFO : Plotted number of elements +rank0 [2026-10-28 12:55:50,767] INFO : Plotted jets +rank0 [2026-10-28 12:55:51,858] INFO : Plotted jet ratio +rank0 [2026-10-28 12:55:51,890] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:55:52,123] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:55:52,516] INFO : Plotted number of elements +rank0 [2026-10-28 12:55:54,416] INFO : Plotted jets +rank0 [2026-10-28 12:55:55,518] INFO : Plotted jet ratio +rank0 [2026-10-28 12:55:55,551] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 12:55:55,766] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 12:55:56,130] INFO : Plotted number of elements +rank0 [2026-10-28 12:55:57,513] INFO : Plotted jets +rank0 [2026-10-28 12:55:58,576] INFO : Plotted jet ratio +rank0 [2026-10-28 12:56:09,574] INFO : Step 37100/50000 rank0 | Train Loss: 8.0368 | Train Loss Unweighted: 3.0025 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6859 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1389 | Regression_eta: 0.0080 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7136 | Classification_binary: 2.6859 | Regression_cos_phi: 0.5275 | Regression_energy: 1.5181 | Regression_eta: 0.3581 | Regression_pt: 1.6172 | Regression_sin_phi: 0.6163 | LR: 2.40e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3213s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5583.5 | uss_MB: 5112.8 | child_rss_MB: 3197.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:09,788] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:20,447] INFO : Step 37200/50000 rank0 | Train Loss: 8.0984 | Train Loss Unweighted: 2.8799 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.5690 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1337 | Regression_eta: 0.0149 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8366 | Classification_binary: 2.5690 | Regression_cos_phi: 0.4368 | Regression_energy: 1.4620 | Regression_eta: 0.6660 | Regression_pt: 1.5060 | Regression_sin_phi: 0.6220 | LR: 2.38e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0216 | max_forward: 0.0147 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5583.5 | uss_MB: 5112.8 | child_rss_MB: 3193.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:56:20,659] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:31,492] INFO : Step 37300/50000 rank0 | Train Loss: 7.7696 | Train Loss Unweighted: 2.9498 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6368 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1390 | Regression_eta: 0.0094 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6913 | Classification_binary: 2.6368 | Regression_cos_phi: 0.3606 | Regression_energy: 1.5196 | Regression_eta: 0.4203 | Regression_pt: 1.6040 | Regression_sin_phi: 0.5368 | LR: 2.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3273s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0798 | max_forward: 0.0144 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5468.9 | uss_MB: 4997.6 | child_rss_MB: 3194.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:31,705] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:42,584] INFO : Step 37400/50000 rank0 | Train Loss: 8.3098 | Train Loss Unweighted: 2.8793 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5563 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1391 | Regression_eta: 0.0273 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6404 | Classification_binary: 2.5563 | Regression_cos_phi: 0.3426 | Regression_energy: 1.5212 | Regression_eta: 1.2185 | Regression_pt: 1.5315 | Regression_sin_phi: 0.4994 | LR: 2.34e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4214s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0862 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1301 | max_tensorboard_logging: 0.3571 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5468.9 | uss_MB: 4997.8 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:42,799] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:53,684] INFO : Step 37500/50000 rank0 | Train Loss: 8.7786 | Train Loss Unweighted: 3.0007 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6424 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1581 | Regression_eta: 0.0114 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7657 | Classification_binary: 2.6424 | Regression_cos_phi: 0.5314 | Regression_energy: 1.7290 | Regression_eta: 0.5106 | Regression_pt: 1.8030 | Regression_sin_phi: 0.7964 | LR: 2.32e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3827s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0337 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:53,895] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:04,769] INFO : Step 37600/50000 rank0 | Train Loss: 7.8635 | Train Loss Unweighted: 2.8732 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5549 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1400 | Regression_eta: 0.0080 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7033 | Classification_binary: 2.5549 | Regression_cos_phi: 0.5197 | Regression_energy: 1.5305 | Regression_eta: 0.3569 | Regression_pt: 1.6358 | Regression_sin_phi: 0.5623 | LR: 2.30e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3877s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0501 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3592 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:57:04,982] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:15,882] INFO : Step 37700/50000 rank0 | Train Loss: 8.1664 | Train Loss Unweighted: 2.8295 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4849 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1535 | Regression_eta: 0.0090 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6949 | Classification_binary: 2.4849 | Regression_cos_phi: 0.4767 | Regression_energy: 1.6786 | Regression_eta: 0.4017 | Regression_pt: 1.7748 | Regression_sin_phi: 0.6549 | LR: 2.28e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3743s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1067 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1290 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:57:16,095] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:26,994] INFO : Step 37800/50000 rank0 | Train Loss: 8.6845 | Train Loss Unweighted: 2.9779 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6273 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1591 | Regression_eta: 0.0163 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6962 | Classification_binary: 2.6273 | Regression_cos_phi: 0.5315 | Regression_energy: 1.7394 | Regression_eta: 0.7274 | Regression_pt: 1.6788 | Regression_sin_phi: 0.6839 | LR: 2.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4881s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1035 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.3687 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:27,206] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:38,042] INFO : Step 37900/50000 rank0 | Train Loss: 8.1660 | Train Loss Unweighted: 2.9079 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5796 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1461 | Regression_eta: 0.0164 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6752 | Classification_binary: 2.5796 | Regression_cos_phi: 0.4300 | Regression_energy: 1.5967 | Regression_eta: 0.7319 | Regression_pt: 1.6110 | Regression_sin_phi: 0.5415 | LR: 2.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3712s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0265 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5469.5 | uss_MB: 4997.8 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:57:38,255] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:49,124] INFO : Step 38000/50000 rank0 | Train Loss: 7.7200 | Train Loss Unweighted: 2.9871 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6836 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1333 | Regression_eta: 0.0073 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6539 | Classification_binary: 2.6836 | Regression_cos_phi: 0.4156 | Regression_energy: 1.4575 | Regression_eta: 0.3257 | Regression_pt: 1.5769 | Regression_sin_phi: 0.6067 | LR: 2.22e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4599s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1012 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3662 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5469.5 | uss_MB: 4997.8 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:57:49,337] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:49,366] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 12:57:49,450] INFO : Running validation on rank0 +rank0 [2026-10-28 12:57:49,450] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:57:49,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:57:50,559] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 12:57:50,706] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:57:50,838] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:57:50,913] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:57:50,982] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:57:51,050] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:57:51,120] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:57:51,190] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:57:51,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:57:51,434] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:57:51,566] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:57:51,637] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:57:51,707] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:57:51,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:57:51,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:57:51,916] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:57:52,032] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:57:52,039] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:57:52,041] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:57:52,044] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:57:52,045] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:57:52,048] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:57:52,049] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:57:52,052] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:57:52,055] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:57:52,056] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:57:52,058] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:57:52,061] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:57:52,062] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:57:52,065] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:57:52,067] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:57:52,069] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:57:58,079] INFO : VALIDATION | Step=38000/50000 | Train Loss=7.7200 | Valid Loss=8.1680 | Train Loss Unweighted=2.9871 | Valid Loss Unweighted=2.9460 | Stale=1 | ETA=33.3m +rank0 [2026-10-28 12:57:58,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:57:58,146] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:57:58,151] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:57:58,323] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:57:59,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:58:00,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:58:01,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:58:01,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:58:01,880] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:58:01,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:58:01,938] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:58:01,941] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:58:02,052] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:58:03,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:58:04,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:58:04,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:58:05,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:58:05,399] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:58:05,402] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:58:05,455] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:58:05,458] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:58:05,589] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:58:07,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:58:07,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:58:08,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:58:08,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:58:08,931] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:58:08,938] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:58:09,205] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:58:09,612] INFO : Plotted number of elements +rank0 [2026-10-28 12:58:11,058] INFO : Plotted jets +rank0 [2026-10-28 12:58:12,150] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:12,183] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:58:12,414] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:58:12,807] INFO : Plotted number of elements +rank0 [2026-10-28 12:58:14,729] INFO : Plotted jets +rank0 [2026-10-28 12:58:15,824] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:15,858] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 12:58:16,069] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 12:58:16,432] INFO : Plotted number of elements +rank0 [2026-10-28 12:58:17,808] INFO : Plotted jets +rank0 [2026-10-28 12:58:18,864] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:30,336] INFO : Step 38100/50000 rank0 | Train Loss: 8.2853 | Train Loss Unweighted: 2.9831 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.6430 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1514 | Regression_eta: 0.0121 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6998 | Classification_binary: 2.6430 | Regression_cos_phi: 0.4179 | Regression_energy: 1.6555 | Regression_eta: 0.5417 | Regression_pt: 1.7245 | Regression_sin_phi: 0.6030 | LR: 2.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4504s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0850 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1340 | max_tensorboard_logging: 0.3593 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5654.0 | uss_MB: 5182.9 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:30,552] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:41,437] INFO : Step 38200/50000 rank0 | Train Loss: 8.4095 | Train Loss Unweighted: 2.9581 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.6136 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1542 | Regression_eta: 0.0107 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6893 | Classification_binary: 2.6136 | Regression_cos_phi: 0.5151 | Regression_energy: 1.6853 | Regression_eta: 0.4763 | Regression_pt: 1.7364 | Regression_sin_phi: 0.6936 | LR: 2.18e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4329s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0606 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5654.0 | uss_MB: 5183.0 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:58:41,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:52,436] INFO : Step 38300/50000 rank0 | Train Loss: 8.1590 | Train Loss Unweighted: 2.9319 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.6187 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1357 | Regression_eta: 0.0108 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6670 | Classification_binary: 2.6187 | Regression_cos_phi: 0.5211 | Regression_energy: 1.4835 | Regression_eta: 0.4803 | Regression_pt: 1.5734 | Regression_sin_phi: 0.8151 | LR: 2.16e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4411s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0141 | max_loss: 0.0116 | max_backward: 0.0335 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5543.8 | uss_MB: 5072.9 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:52,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:03,615] INFO : Step 38400/50000 rank0 | Train Loss: 7.6877 | Train Loss Unweighted: 2.8553 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5441 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1371 | Regression_eta: 0.0100 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6736 | Classification_binary: 2.5441 | Regression_cos_phi: 0.3376 | Regression_energy: 1.4991 | Regression_eta: 0.4462 | Regression_pt: 1.6014 | Regression_sin_phi: 0.5857 | LR: 2.14e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4341s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0945 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.3617 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5543.8 | uss_MB: 5073.5 | child_rss_MB: 3200.2 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:03,825] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:14,739] INFO : Step 38500/50000 rank0 | Train Loss: 8.3172 | Train Loss Unweighted: 2.9679 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6263 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1511 | Regression_eta: 0.0102 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7284 | Classification_binary: 2.6263 | Regression_cos_phi: 0.4139 | Regression_energy: 1.6514 | Regression_eta: 0.4549 | Regression_pt: 1.7503 | Regression_sin_phi: 0.6918 | LR: 2.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3742s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.3625 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5544.8 | uss_MB: 5073.6 | child_rss_MB: 3196.2 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:14,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:25,786] INFO : Step 38600/50000 rank0 | Train Loss: 7.7003 | Train Loss Unweighted: 2.8752 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5711 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1335 | Regression_eta: 0.0102 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6900 | Classification_binary: 2.5711 | Regression_cos_phi: 0.4256 | Regression_energy: 1.4600 | Regression_eta: 0.4559 | Regression_pt: 1.5392 | Regression_sin_phi: 0.5585 | LR: 2.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4849s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0473 | max_backward: 0.0332 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3890 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5544.8 | uss_MB: 5073.6 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:59:25,997] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:36,955] INFO : Step 38700/50000 rank0 | Train Loss: 8.0046 | Train Loss Unweighted: 2.9557 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6340 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1404 | Regression_eta: 0.0123 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6732 | Classification_binary: 2.6340 | Regression_cos_phi: 0.3256 | Regression_energy: 1.5349 | Regression_eta: 0.5496 | Regression_pt: 1.6565 | Regression_sin_phi: 0.6307 | LR: 2.09e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0793 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.3833 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5544.8 | uss_MB: 5073.7 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:37,166] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:47,938] INFO : Step 38800/50000 rank0 | Train Loss: 7.9986 | Train Loss Unweighted: 2.8914 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5696 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1377 | Regression_eta: 0.0129 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7049 | Classification_binary: 2.5696 | Regression_cos_phi: 0.3796 | Regression_energy: 1.5058 | Regression_eta: 0.5772 | Regression_pt: 1.6664 | Regression_sin_phi: 0.5950 | LR: 2.07e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0394 | max_backward: 0.0333 | max_optimizer_step: 0.1301 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5544.8 | uss_MB: 5073.8 | child_rss_MB: 3197.8 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:59:48,152] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:59,184] INFO : Step 38900/50000 rank0 | Train Loss: 7.8843 | Train Loss Unweighted: 2.9190 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5996 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1413 | Regression_eta: 0.0088 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7085 | Classification_binary: 2.5996 | Regression_cos_phi: 0.3719 | Regression_energy: 1.5450 | Regression_eta: 0.3905 | Regression_pt: 1.6413 | Regression_sin_phi: 0.6276 | LR: 2.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3910s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0838 | max_forward: 0.0153 | max_loss: 0.0035 | max_backward: 0.0344 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5544.8 | uss_MB: 5072.6 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:59:59,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:10,298] INFO : Step 39000/50000 rank0 | Train Loss: 7.4532 | Train Loss Unweighted: 2.7462 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4544 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1258 | Regression_eta: 0.0098 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7320 | Classification_binary: 2.4544 | Regression_cos_phi: 0.3759 | Regression_energy: 1.3749 | Regression_eta: 0.4364 | Regression_pt: 1.4815 | Regression_sin_phi: 0.5980 | LR: 2.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3844s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0561 | max_forward: 0.0176 | max_loss: 0.0034 | max_backward: 0.0390 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.2962 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5545.8 | uss_MB: 5075.8 | child_rss_MB: 3195.9 | allocated_MB: 210.7 | reserved_MB: 2852.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:00:10,514] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:10,533] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 13:00:10,616] INFO : Running validation on rank0 +rank0 [2026-10-28 13:00:10,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:00:10,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:00:11,708] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 13:00:11,863] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:00:11,996] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:00:12,070] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:00:12,140] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:00:12,208] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:00:12,277] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:00:12,347] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:00:12,457] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:00:12,590] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:00:12,721] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:00:12,793] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:00:12,863] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:00:12,931] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:00:13,000] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:00:13,070] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:00:13,179] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:00:13,186] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:00:13,188] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:00:13,296] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:00:13,299] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:00:13,301] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:00:13,302] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:00:13,305] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:00:13,308] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:00:13,309] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:00:13,312] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:00:13,314] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:00:13,315] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:00:13,318] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:00:13,320] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:00:13,321] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:00:19,286] INFO : VALIDATION | Step=39000/50000 | Train Loss=7.4532 | Valid Loss=8.0735 | Train Loss Unweighted=2.7462 | Valid Loss Unweighted=2.9264 | Stale=0 | ETA=30.4m +rank0 [2026-10-28 13:00:19,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:19,353] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:00:19,356] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:00:19,529] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:21,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:21,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:22,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:22,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:23,050] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:23,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:23,109] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:00:23,112] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:23,233] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:24,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:25,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:25,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:26,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:26,592] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:26,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:26,650] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:00:26,653] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:00:26,769] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:28,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:28,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:29,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:29,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:30,143] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:30,150] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:00:30,413] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:00:30,822] INFO : Plotted number of elements +rank0 [2026-10-28 13:00:32,271] INFO : Plotted jets +rank0 [2026-10-28 13:00:33,357] INFO : Plotted jet ratio +rank0 [2026-10-28 13:00:33,388] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:33,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:34,016] INFO : Plotted number of elements +rank0 [2026-10-28 13:00:35,413] INFO : Plotted jets +rank0 [2026-10-28 13:00:37,044] INFO : Plotted jet ratio +rank0 [2026-10-28 13:00:37,076] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 13:00:37,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 13:00:37,651] INFO : Plotted number of elements +rank0 [2026-10-28 13:00:39,030] INFO : Plotted jets +rank0 [2026-10-28 13:00:40,090] INFO : Plotted jet ratio +rank0 [2026-10-28 13:00:51,406] INFO : Step 39100/50000 rank0 | Train Loss: 8.3178 | Train Loss Unweighted: 2.9700 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6354 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1473 | Regression_eta: 0.0116 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7533 | Classification_binary: 2.6354 | Regression_cos_phi: 0.4263 | Regression_energy: 1.6102 | Regression_eta: 0.5162 | Regression_pt: 1.6863 | Regression_sin_phi: 0.6902 | LR: 2.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5076s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0167 | max_loss: 0.1081 | max_backward: 0.0383 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3865 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5735.9 | uss_MB: 5265.9 | child_rss_MB: 3193.8 | allocated_MB: 210.7 | reserved_MB: 2856.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:00:51,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:02,623] INFO : Step 39200/50000 rank0 | Train Loss: 8.0174 | Train Loss Unweighted: 2.9415 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6139 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1448 | Regression_eta: 0.0087 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7309 | Classification_binary: 2.6139 | Regression_cos_phi: 0.4513 | Regression_energy: 1.5825 | Regression_eta: 0.3889 | Regression_pt: 1.6849 | Regression_sin_phi: 0.5650 | LR: 2.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4167s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0557 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.3686 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5735.9 | uss_MB: 5263.8 | child_rss_MB: 3196.5 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:02,838] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:13,644] INFO : Step 39300/50000 rank0 | Train Loss: 8.4233 | Train Loss Unweighted: 2.9568 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6039 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1587 | Regression_eta: 0.0096 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6746 | Classification_binary: 2.6039 | Regression_cos_phi: 0.4954 | Regression_energy: 1.7354 | Regression_eta: 0.4265 | Regression_pt: 1.8038 | Regression_sin_phi: 0.6838 | LR: 1.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4850s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0766 | max_backward: 0.0331 | max_optimizer_step: 0.1296 | max_tensorboard_logging: 0.4056 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5616.3 | uss_MB: 5144.5 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:13,856] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:24,845] INFO : Step 39400/50000 rank0 | Train Loss: 7.4922 | Train Loss Unweighted: 2.8894 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5917 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1321 | Regression_eta: 0.0098 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6487 | Classification_binary: 2.5917 | Regression_cos_phi: 0.3331 | Regression_energy: 1.4444 | Regression_eta: 0.4386 | Regression_pt: 1.5201 | Regression_sin_phi: 0.5155 | LR: 1.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3249s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0554 | max_forward: 0.0141 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1308 | max_tensorboard_logging: 0.2774 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5617.3 | uss_MB: 5145.6 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:25,060] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:35,822] INFO : Step 39500/50000 rank0 | Train Loss: 7.9355 | Train Loss Unweighted: 2.8580 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5313 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1480 | Regression_eta: 0.0103 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6459 | Classification_binary: 2.5313 | Regression_cos_phi: 0.4707 | Regression_energy: 1.6177 | Regression_eta: 0.4599 | Regression_pt: 1.6402 | Regression_sin_phi: 0.5698 | LR: 1.94e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.4741s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0980 | max_backward: 0.0333 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5617.3 | uss_MB: 5145.5 | child_rss_MB: 3193.6 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:01:36,036] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:47,087] INFO : Step 39600/50000 rank0 | Train Loss: 7.9733 | Train Loss Unweighted: 2.8895 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5723 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1389 | Regression_eta: 0.0104 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6768 | Classification_binary: 2.5723 | Regression_cos_phi: 0.4617 | Regression_energy: 1.5186 | Regression_eta: 0.4658 | Regression_pt: 1.6129 | Regression_sin_phi: 0.6651 | LR: 1.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3918s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0902 | max_forward: 0.0162 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1324 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5618.3 | uss_MB: 5148.1 | child_rss_MB: 3194.2 | allocated_MB: 210.7 | reserved_MB: 2856.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:01:47,300] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:58,026] INFO : Step 39700/50000 rank0 | Train Loss: 8.5092 | Train Loss Unweighted: 2.8995 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5586 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1500 | Regression_eta: 0.0194 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6866 | Classification_binary: 2.5586 | Regression_cos_phi: 0.4839 | Regression_energy: 1.6401 | Regression_eta: 0.8651 | Regression_pt: 1.6537 | Regression_sin_phi: 0.6212 | LR: 1.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4182s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0289 | max_backward: 0.0333 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.3823 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5618.3 | uss_MB: 5146.1 | child_rss_MB: 3191.7 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:01:58,237] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:09,293] INFO : Step 39800/50000 rank0 | Train Loss: 8.4283 | Train Loss Unweighted: 3.0459 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6965 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1562 | Regression_eta: 0.0118 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7605 | Classification_binary: 2.6965 | Regression_cos_phi: 0.3720 | Regression_energy: 1.7077 | Regression_eta: 0.5258 | Regression_pt: 1.7734 | Regression_sin_phi: 0.5925 | LR: 1.89e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3995s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0925 | max_forward: 0.0185 | max_loss: 0.0034 | max_backward: 0.0394 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.2895 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5619.3 | uss_MB: 5148.3 | child_rss_MB: 3190.8 | allocated_MB: 210.7 | reserved_MB: 2856.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:02:09,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:20,355] INFO : Step 39900/50000 rank0 | Train Loss: 8.0010 | Train Loss Unweighted: 2.7597 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4518 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1326 | Regression_eta: 0.0146 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7201 | Classification_binary: 2.4518 | Regression_cos_phi: 0.4048 | Regression_energy: 1.4501 | Regression_eta: 0.6533 | Regression_pt: 1.5053 | Regression_sin_phi: 0.8156 | LR: 1.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4477s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1320 | max_tensorboard_logging: 0.3788 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5618.0 | uss_MB: 5144.6 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:02:20,571] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:31,433] INFO : Step 40000/50000 rank0 | Train Loss: 8.1728 | Train Loss Unweighted: 2.9375 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5997 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1469 | Regression_eta: 0.0161 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6564 | Classification_binary: 2.5997 | Regression_cos_phi: 0.3214 | Regression_energy: 1.6063 | Regression_eta: 0.7199 | Regression_pt: 1.7440 | Regression_sin_phi: 0.5251 | LR: 1.86e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0813 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3497 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5619.0 | uss_MB: 5145.7 | child_rss_MB: 3192.1 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:02:31,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:31,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 13:02:31,768] INFO : Running validation on rank0 +rank0 [2026-10-28 13:02:31,768] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:02:31,768] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:02:32,861] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 13:02:33,008] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:02:33,140] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:02:33,215] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:02:33,284] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:02:33,352] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:02:33,421] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:02:33,491] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:02:33,600] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:02:33,733] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:02:33,865] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:02:33,936] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:02:34,006] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:02:34,075] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:02:34,145] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:02:34,214] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:02:34,324] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:02:34,331] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:02:34,334] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:02:34,336] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:02:34,337] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:02:34,340] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:02:34,343] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:02:34,344] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:02:34,347] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:02:34,350] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:02:34,351] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:02:34,354] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:02:34,357] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:02:34,359] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:02:34,360] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:02:34,363] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:02:40,157] INFO : VALIDATION | Step=40000/50000 | Train Loss=8.1728 | Valid Loss=8.0892 | Train Loss Unweighted=2.9375 | Valid Loss Unweighted=2.9256 | Stale=1 | ETA=27.5m +rank0 [2026-10-28 13:02:40,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:02:40,291] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:02:40,295] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:02:40,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:02:42,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:02:42,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:02:43,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:02:44,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:02:44,233] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:02:44,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:02:44,290] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:02:44,294] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:02:44,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:02:45,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:02:46,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:02:47,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:02:47,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:02:47,824] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:02:47,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:02:47,879] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:02:47,882] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:02:48,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:02:49,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:02:50,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:02:50,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:02:51,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:02:51,352] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:02:51,360] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:02:51,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:02:52,027] INFO : Plotted number of elements +rank0 [2026-10-28 13:02:53,476] INFO : Plotted jets +rank0 [2026-10-28 13:02:54,558] INFO : Plotted jet ratio +rank0 [2026-10-28 13:02:54,589] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:02:54,818] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:02:55,210] INFO : Plotted number of elements +rank0 [2026-10-28 13:02:57,141] INFO : Plotted jets +rank0 [2026-10-28 13:02:58,239] INFO : Plotted jet ratio +rank0 [2026-10-28 13:02:58,271] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 13:02:58,486] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 13:02:58,847] INFO : Plotted number of elements +rank0 [2026-10-28 13:03:00,227] INFO : Plotted jets +rank0 [2026-10-28 13:03:01,301] INFO : Plotted jet ratio +rank0 [2026-10-28 13:03:12,453] INFO : Step 40100/50000 rank0 | Train Loss: 7.3796 | Train Loss Unweighted: 2.8598 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.5761 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1244 | Regression_eta: 0.0095 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.5784 | Classification_binary: 2.5761 | Regression_cos_phi: 0.4077 | Regression_energy: 1.3605 | Regression_eta: 0.4243 | Regression_pt: 1.4485 | Regression_sin_phi: 0.5840 | LR: 1.84e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3858s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0139 | max_loss: 0.0034 | max_backward: 0.1218 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5801.9 | uss_MB: 5327.4 | child_rss_MB: 3225.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:03:12,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:23,772] INFO : Step 40200/50000 rank0 | Train Loss: 7.9818 | Train Loss Unweighted: 2.9835 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6717 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1376 | Regression_eta: 0.0085 | Regression_pt: 0.1331 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7140 | Classification_binary: 2.6717 | Regression_cos_phi: 0.3563 | Regression_energy: 1.5040 | Regression_eta: 0.3809 | Regression_pt: 1.5813 | Regression_sin_phi: 0.7737 | LR: 1.83e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4965s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0987 | max_backward: 0.0334 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5801.9 | uss_MB: 5327.6 | child_rss_MB: 3194.6 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:23,984] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:34,674] INFO : Step 40300/50000 rank0 | Train Loss: 7.2253 | Train Loss Unweighted: 2.8859 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.6023 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1255 | Regression_eta: 0.0062 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6369 | Classification_binary: 2.6023 | Regression_cos_phi: 0.3550 | Regression_energy: 1.3719 | Regression_eta: 0.2772 | Regression_pt: 1.4757 | Regression_sin_phi: 0.5064 | LR: 1.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0896 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5801.9 | uss_MB: 5327.4 | child_rss_MB: 3221.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:03:34,890] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:46,055] INFO : Step 40400/50000 rank0 | Train Loss: 7.1136 | Train Loss Unweighted: 2.6160 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3425 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1191 | Regression_eta: 0.0108 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6879 | Classification_binary: 2.3425 | Regression_cos_phi: 0.4172 | Regression_energy: 1.3018 | Regression_eta: 0.4796 | Regression_pt: 1.3452 | Regression_sin_phi: 0.5394 | LR: 1.79e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4704s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1165 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.3516 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5693.0 | uss_MB: 5219.8 | child_rss_MB: 3197.1 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:03:46,267] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:57,096] INFO : Step 40500/50000 rank0 | Train Loss: 8.5539 | Train Loss Unweighted: 2.8966 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5351 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1618 | Regression_eta: 0.0144 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7160 | Classification_binary: 2.5351 | Regression_cos_phi: 0.4158 | Regression_energy: 1.7692 | Regression_eta: 0.6432 | Regression_pt: 1.8169 | Regression_sin_phi: 0.6576 | LR: 1.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5112s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0187 | max_loss: 0.0713 | max_backward: 0.0389 | max_optimizer_step: 0.1324 | max_tensorboard_logging: 0.3871 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5693.0 | uss_MB: 5222.3 | child_rss_MB: 3199.0 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:03:57,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:08,287] INFO : Step 40600/50000 rank0 | Train Loss: 7.9018 | Train Loss Unweighted: 2.7847 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4720 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1383 | Regression_eta: 0.0142 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6668 | Classification_binary: 2.4720 | Regression_cos_phi: 0.3798 | Regression_energy: 1.5121 | Regression_eta: 0.6324 | Regression_pt: 1.5354 | Regression_sin_phi: 0.7033 | LR: 1.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3252s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0209 | max_forward: 0.0159 | max_loss: 0.0035 | max_backward: 0.0382 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5693.0 | uss_MB: 5222.4 | child_rss_MB: 3198.7 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:04:08,503] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:19,321] INFO : Step 40700/50000 rank0 | Train Loss: 7.9515 | Train Loss Unweighted: 2.8729 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5529 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1409 | Regression_eta: 0.0096 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7074 | Classification_binary: 2.5529 | Regression_cos_phi: 0.5130 | Regression_energy: 1.5407 | Regression_eta: 0.4297 | Regression_pt: 1.6226 | Regression_sin_phi: 0.5852 | LR: 1.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4502s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0998 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3435 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5693.0 | uss_MB: 5220.3 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:04:19,534] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:30,358] INFO : Step 40800/50000 rank0 | Train Loss: 7.5936 | Train Loss Unweighted: 2.8754 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5823 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1279 | Regression_eta: 0.0085 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6978 | Classification_binary: 2.5823 | Regression_cos_phi: 0.4779 | Regression_energy: 1.3985 | Regression_eta: 0.3798 | Regression_pt: 1.4816 | Regression_sin_phi: 0.5757 | LR: 1.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0522 | max_backward: 0.0343 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.3728 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5693.0 | uss_MB: 5220.4 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:04:30,574] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:41,459] INFO : Step 40900/50000 rank0 | Train Loss: 8.3018 | Train Loss Unweighted: 2.9003 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5792 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1388 | Regression_eta: 0.0163 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7453 | Classification_binary: 2.5792 | Regression_cos_phi: 0.5053 | Regression_energy: 1.5174 | Regression_eta: 0.7281 | Regression_pt: 1.5610 | Regression_sin_phi: 0.6656 | LR: 1.72e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3964s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0172 | max_loss: 0.0055 | max_backward: 0.0385 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5693.0 | uss_MB: 5222.6 | child_rss_MB: 3198.6 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 13:04:41,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:52,529] INFO : Step 41000/50000 rank0 | Train Loss: 8.6550 | Train Loss Unweighted: 3.0564 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7006 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1555 | Regression_eta: 0.0107 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8521 | Classification_binary: 2.7006 | Regression_cos_phi: 0.4237 | Regression_energy: 1.6996 | Regression_eta: 0.4794 | Regression_pt: 1.8206 | Regression_sin_phi: 0.6791 | LR: 1.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0476 | max_backward: 0.0335 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3701 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5692.3 | uss_MB: 5219.8 | child_rss_MB: 3199.0 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:04:52,741] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:52,758] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 13:04:52,943] INFO : Running validation on rank0 +rank0 [2026-10-28 13:04:52,943] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:04:52,943] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:04:54,044] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 13:04:54,190] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:04:54,324] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:04:54,399] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:04:54,468] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:04:54,537] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:04:54,607] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:04:54,676] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:04:54,785] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:04:54,917] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:04:55,049] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:04:55,121] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:04:55,191] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:04:55,259] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:04:55,329] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:04:55,399] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:04:55,507] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:04:55,514] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:04:55,517] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:04:55,520] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:04:55,522] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:04:55,524] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:04:55,528] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:04:55,530] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:04:55,531] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:04:55,534] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:04:55,535] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:04:55,537] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:04:55,540] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:04:55,543] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:04:55,546] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:04:55,547] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:05:01,607] INFO : VALIDATION | Step=41000/50000 | Train Loss=8.6550 | Valid Loss=8.0760 | Train Loss Unweighted=3.0564 | Valid Loss Unweighted=2.9276 | Stale=2 | ETA=24.7m +rank0 [2026-10-28 13:05:01,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:05:01,708] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:05:01,711] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:05:01,889] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:05:03,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:05:04,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:05:04,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:05:05,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:05:05,411] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:05:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:05:05,468] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:05:05,471] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:05:05,601] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:05:07,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:05:07,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:05:08,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:05:08,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:05:08,983] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:05:08,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:05:09,040] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:05:09,042] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:05:09,163] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:05:10,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:05:11,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:05:11,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:05:12,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:05:12,466] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:05:12,474] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:05:12,734] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:05:13,139] INFO : Plotted number of elements +rank0 [2026-10-28 13:05:14,583] INFO : Plotted jets +rank0 [2026-10-28 13:05:15,668] INFO : Plotted jet ratio +rank0 [2026-10-28 13:05:15,701] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:05:15,931] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:05:16,327] INFO : Plotted number of elements +rank0 [2026-10-28 13:05:17,727] INFO : Plotted jets +rank0 [2026-10-28 13:05:19,386] INFO : Plotted jet ratio +rank0 [2026-10-28 13:05:19,419] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 13:05:19,628] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 13:05:19,989] INFO : Plotted number of elements +rank0 [2026-10-28 13:05:21,366] INFO : Plotted jets +rank0 [2026-10-28 13:05:22,423] INFO : Plotted jet ratio +rank0 [2026-10-28 13:05:33,680] INFO : Step 41100/50000 rank0 | Train Loss: 8.1373 | Train Loss Unweighted: 2.8312 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5183 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1347 | Regression_eta: 0.0121 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7086 | Classification_binary: 2.5183 | Regression_cos_phi: 0.5720 | Regression_energy: 1.4732 | Regression_eta: 0.5395 | Regression_pt: 1.5477 | Regression_sin_phi: 0.7782 | LR: 1.69e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3481s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0503 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0382 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.2741 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5885.6 | uss_MB: 5416.9 | child_rss_MB: 3194.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:05:33,896] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:44,800] INFO : Step 41200/50000 rank0 | Train Loss: 8.3790 | Train Loss Unweighted: 2.8729 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5341 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1496 | Regression_eta: 0.0146 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7315 | Classification_binary: 2.5341 | Regression_cos_phi: 0.5241 | Regression_energy: 1.6352 | Regression_eta: 0.6512 | Regression_pt: 1.6690 | Regression_sin_phi: 0.6340 | LR: 1.67e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0695 | max_forward: 0.0149 | max_loss: 0.0041 | max_backward: 0.0332 | max_optimizer_step: 0.1368 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5885.6 | uss_MB: 5414.8 | child_rss_MB: 3192.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:05:45,017] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:55,777] INFO : Step 41300/50000 rank0 | Train Loss: 8.2107 | Train Loss Unweighted: 2.8333 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5127 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1390 | Regression_eta: 0.0151 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7121 | Classification_binary: 2.5127 | Regression_cos_phi: 0.4974 | Regression_energy: 1.5194 | Regression_eta: 0.6723 | Regression_pt: 1.5726 | Regression_sin_phi: 0.7243 | LR: 1.66e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4399s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0199 | max_backward: 0.0333 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3806 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5885.6 | uss_MB: 5414.8 | child_rss_MB: 3190.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:05:55,996] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:06,958] INFO : Step 41400/50000 rank0 | Train Loss: 7.6895 | Train Loss Unweighted: 2.8100 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5063 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1327 | Regression_eta: 0.0120 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6824 | Classification_binary: 2.5063 | Regression_cos_phi: 0.4628 | Regression_energy: 1.4503 | Regression_eta: 0.5372 | Regression_pt: 1.5231 | Regression_sin_phi: 0.5274 | LR: 1.64e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4366s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0602 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3622 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5763.0 | uss_MB: 5291.4 | child_rss_MB: 3188.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:06:07,172] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:18,054] INFO : Step 41500/50000 rank0 | Train Loss: 7.8747 | Train Loss Unweighted: 2.8599 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5318 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1456 | Regression_eta: 0.0096 | Regression_pt: 0.1429 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6907 | Classification_binary: 2.5318 | Regression_cos_phi: 0.3700 | Regression_energy: 1.5920 | Regression_eta: 0.4278 | Regression_pt: 1.6980 | Regression_sin_phi: 0.5644 | LR: 1.63e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3830s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0193 | max_loss: 0.0036 | max_backward: 0.0336 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5764.0 | uss_MB: 5291.6 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:06:18,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:29,208] INFO : Step 41600/50000 rank0 | Train Loss: 7.8484 | Train Loss Unweighted: 2.8038 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4822 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1430 | Regression_eta: 0.0086 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6642 | Classification_binary: 2.4822 | Regression_cos_phi: 0.4434 | Regression_energy: 1.5632 | Regression_eta: 0.3856 | Regression_pt: 1.6452 | Regression_sin_phi: 0.6646 | LR: 1.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3752s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0503 | max_forward: 0.0144 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5764.0 | uss_MB: 5291.7 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 33.0 +rank0 [2026-10-28 13:06:29,422] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:40,206] INFO : Step 41700/50000 rank0 | Train Loss: 7.9916 | Train Loss Unweighted: 2.9344 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6126 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1439 | Regression_eta: 0.0114 | Regression_pt: 0.1351 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7186 | Classification_binary: 2.6126 | Regression_cos_phi: 0.4079 | Regression_energy: 1.5731 | Regression_eta: 0.5103 | Regression_pt: 1.6059 | Regression_sin_phi: 0.5632 | LR: 1.60e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4323s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0844 | max_loss: 0.0034 | max_backward: 0.0390 | max_optimizer_step: 0.1319 | max_tensorboard_logging: 0.3462 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5764.0 | uss_MB: 5293.8 | child_rss_MB: 3188.2 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:06:40,419] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:51,384] INFO : Step 41800/50000 rank0 | Train Loss: 7.8281 | Train Loss Unweighted: 2.9358 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.6259 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1391 | Regression_eta: 0.0104 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6169 | Classification_binary: 2.6259 | Regression_cos_phi: 0.3886 | Regression_energy: 1.5206 | Regression_eta: 0.4651 | Regression_pt: 1.5570 | Regression_sin_phi: 0.6540 | LR: 1.58e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4128s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0848 | max_forward: 0.0136 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0322 | Memory: rss_MB: 5764.0 | uss_MB: 5291.7 | child_rss_MB: 3190.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:06:51,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:02,321] INFO : Step 41900/50000 rank0 | Train Loss: 7.6639 | Train Loss Unweighted: 2.9224 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.6326 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1270 | Regression_eta: 0.0098 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6250 | Classification_binary: 2.6326 | Regression_cos_phi: 0.4188 | Regression_energy: 1.3882 | Regression_eta: 0.4378 | Regression_pt: 1.4559 | Regression_sin_phi: 0.7054 | LR: 1.57e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5093s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0883 | max_backward: 0.0339 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.3907 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5764.0 | uss_MB: 5291.8 | child_rss_MB: 3190.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:07:02,535] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:13,523] INFO : Step 42000/50000 rank0 | Train Loss: 8.1711 | Train Loss Unweighted: 2.8826 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5551 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1438 | Regression_eta: 0.0126 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7149 | Classification_binary: 2.5551 | Regression_cos_phi: 0.4415 | Regression_energy: 1.5726 | Regression_eta: 0.5632 | Regression_pt: 1.6402 | Regression_sin_phi: 0.6836 | LR: 1.56e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3806s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0939 | max_forward: 0.0137 | max_loss: 0.0036 | max_backward: 0.0337 | max_optimizer_step: 0.1320 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5763.9 | uss_MB: 5291.5 | child_rss_MB: 3188.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:07:13,736] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:13,755] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 13:07:13,837] INFO : Running validation on rank0 +rank0 [2026-10-28 13:07:13,837] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:07:13,837] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:07:14,989] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 13:07:15,137] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:07:15,270] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:07:15,345] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:07:15,413] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:07:15,482] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:07:15,553] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:07:15,624] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:07:15,732] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:07:15,866] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:07:15,999] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:07:16,071] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:07:16,141] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:07:16,210] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:07:16,280] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:07:16,350] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:07:16,462] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:07:16,568] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:07:16,571] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:07:16,575] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:07:16,577] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:07:16,578] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:07:16,581] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:07:16,584] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:07:16,585] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:07:16,588] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:07:16,591] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:07:16,593] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:07:16,594] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:07:16,597] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:07:16,599] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:07:16,600] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:07:22,568] INFO : VALIDATION | Step=42000/50000 | Train Loss=8.1711 | Valid Loss=7.9845 | Train Loss Unweighted=2.8826 | Valid Loss Unweighted=2.8902 | Stale=0 | ETA=21.9m +rank0 [2026-10-28 13:07:22,577] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:07:22,634] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:07:22,637] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:07:22,818] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:07:24,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:07:24,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:07:25,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:07:26,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:07:26,362] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:07:26,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:07:26,421] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:07:26,424] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:07:26,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:07:28,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:07:28,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:07:29,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:07:29,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:07:29,933] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:07:29,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:07:29,989] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:07:29,992] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:07:30,109] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:07:31,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:07:32,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:07:32,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:07:33,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:07:33,520] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:07:33,528] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:07:33,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:07:34,195] INFO : Plotted number of elements +rank0 [2026-10-28 13:07:35,638] INFO : Plotted jets +rank0 [2026-10-28 13:07:36,729] INFO : Plotted jet ratio +rank0 [2026-10-28 13:07:36,761] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:07:36,996] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:07:37,390] INFO : Plotted number of elements +rank0 [2026-10-28 13:07:38,784] INFO : Plotted jets +rank0 [2026-10-28 13:07:40,420] INFO : Plotted jet ratio +rank0 [2026-10-28 13:07:40,452] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 13:07:40,659] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 13:07:41,022] INFO : Plotted number of elements +rank0 [2026-10-28 13:07:42,402] INFO : Plotted jets +rank0 [2026-10-28 13:07:43,465] INFO : Plotted jet ratio +rank0 [2026-10-28 13:07:54,893] INFO : Step 42100/50000 rank0 | Train Loss: 7.6065 | Train Loss Unweighted: 2.8897 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5948 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1303 | Regression_eta: 0.0100 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6966 | Classification_binary: 2.5948 | Regression_cos_phi: 0.4638 | Regression_energy: 1.4242 | Regression_eta: 0.4478 | Regression_pt: 1.4672 | Regression_sin_phi: 0.5121 | LR: 1.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4430s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0519 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.3420 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5957.4 | uss_MB: 5486.3 | child_rss_MB: 3192.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:07:55,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:05,858] INFO : Step 42200/50000 rank0 | Train Loss: 8.1083 | Train Loss Unweighted: 2.8238 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5035 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1383 | Regression_eta: 0.0150 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6897 | Classification_binary: 2.5035 | Regression_cos_phi: 0.3945 | Regression_energy: 1.5125 | Regression_eta: 0.6692 | Regression_pt: 1.6006 | Regression_sin_phi: 0.7383 | LR: 1.53e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.4263s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0429 | max_backward: 0.0334 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3697 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5957.4 | uss_MB: 5486.2 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:08:06,074] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:08,526] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 13:08:08,526] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:08:17,436] INFO : Step 42300/50000 rank0 | Train Loss: 7.4707 | Train Loss Unweighted: 2.8143 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5195 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1305 | Regression_eta: 0.0082 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6692 | Classification_binary: 2.5195 | Regression_cos_phi: 0.3910 | Regression_energy: 1.4263 | Regression_eta: 0.3647 | Regression_pt: 1.4964 | Regression_sin_phi: 0.6036 | LR: 1.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4312s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.1401 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5987.6 | uss_MB: 5516.5 | child_rss_MB: 3223.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:08:17,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:28,761] INFO : Step 42400/50000 rank0 | Train Loss: 8.2913 | Train Loss Unweighted: 2.9162 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5942 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1398 | Regression_eta: 0.0182 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7093 | Classification_binary: 2.5942 | Regression_cos_phi: 0.4048 | Regression_energy: 1.5284 | Regression_eta: 0.8122 | Regression_pt: 1.5659 | Regression_sin_phi: 0.6765 | LR: 1.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5081s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0175 | max_loss: 0.0981 | max_backward: 0.0389 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5831.4 | uss_MB: 5362.5 | child_rss_MB: 3197.4 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:08:28,971] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:39,847] INFO : Step 42500/50000 rank0 | Train Loss: 7.7551 | Train Loss Unweighted: 2.8437 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5381 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1339 | Regression_eta: 0.0095 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6842 | Classification_binary: 2.5381 | Regression_cos_phi: 0.4348 | Regression_energy: 1.4644 | Regression_eta: 0.4230 | Regression_pt: 1.5484 | Regression_sin_phi: 0.6622 | LR: 1.49e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4482s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0145 | max_loss: 0.0519 | max_backward: 0.0333 | max_optimizer_step: 0.1337 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5831.4 | uss_MB: 5360.9 | child_rss_MB: 3195.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:08:40,060] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:50,842] INFO : Step 42600/50000 rank0 | Train Loss: 8.1239 | Train Loss Unweighted: 2.9165 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6068 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1339 | Regression_eta: 0.0151 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6559 | Classification_binary: 2.6068 | Regression_cos_phi: 0.4255 | Regression_energy: 1.4642 | Regression_eta: 0.6722 | Regression_pt: 1.5282 | Regression_sin_phi: 0.7712 | LR: 1.48e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4265s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0166 | max_loss: 0.0033 | max_backward: 0.0514 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.2862 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5831.4 | uss_MB: 5361.8 | child_rss_MB: 3193.2 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 13:08:51,055] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:02,028] INFO : Step 42700/50000 rank0 | Train Loss: 7.7339 | Train Loss Unweighted: 2.8490 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5431 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1343 | Regression_eta: 0.0104 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6388 | Classification_binary: 2.5431 | Regression_cos_phi: 0.4663 | Regression_energy: 1.4682 | Regression_eta: 0.4652 | Regression_pt: 1.5533 | Regression_sin_phi: 0.5989 | LR: 1.47e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4216s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0097 | max_backward: 0.0332 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.3770 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5831.4 | uss_MB: 5361.2 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:02,238] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:13,228] INFO : Step 42800/50000 rank0 | Train Loss: 7.6142 | Train Loss Unweighted: 2.8362 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.5373 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1337 | Regression_eta: 0.0128 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6119 | Classification_binary: 2.5373 | Regression_cos_phi: 0.3838 | Regression_energy: 1.4613 | Regression_eta: 0.5700 | Regression_pt: 1.4759 | Regression_sin_phi: 0.5741 | LR: 1.45e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4314s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0755 | max_loss: 0.0037 | max_backward: 0.0335 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3591 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5831.4 | uss_MB: 5361.3 | child_rss_MB: 3196.4 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:09:13,439] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:24,000] INFO : Step 42900/50000 rank0 | Train Loss: 7.5191 | Train Loss Unweighted: 2.8722 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5714 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1325 | Regression_eta: 0.0083 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6391 | Classification_binary: 2.5714 | Regression_cos_phi: 0.3682 | Regression_energy: 1.4482 | Regression_eta: 0.3705 | Regression_pt: 1.5609 | Regression_sin_phi: 0.5607 | LR: 1.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3230s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0706 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.2722 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5831.4 | uss_MB: 5361.3 | child_rss_MB: 3225.3 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:24,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:35,101] INFO : Step 43000/50000 rank0 | Train Loss: 8.0818 | Train Loss Unweighted: 2.9204 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6062 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1372 | Regression_eta: 0.0107 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6932 | Classification_binary: 2.6062 | Regression_cos_phi: 0.4658 | Regression_energy: 1.4998 | Regression_eta: 0.4794 | Regression_pt: 1.5760 | Regression_sin_phi: 0.7613 | LR: 1.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3479s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0035 | max_backward: 0.0752 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.2797 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5830.4 | uss_MB: 5359.1 | child_rss_MB: 3222.8 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:35,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:35,332] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 13:09:35,415] INFO : Running validation on rank0 +rank0 [2026-10-28 13:09:35,415] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:09:35,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:09:36,596] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 13:09:36,742] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:09:36,875] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:09:36,950] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:09:37,020] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:09:37,088] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:09:37,159] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:09:37,230] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:09:37,342] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:09:37,475] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:09:37,607] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:09:37,680] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:09:37,750] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:09:37,819] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:09:37,890] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:09:37,961] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:09:38,076] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:09:38,083] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:09:38,086] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:09:38,089] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:09:38,091] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:09:38,094] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:09:38,097] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:09:38,099] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:09:38,102] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:09:38,103] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:09:38,105] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:09:38,106] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:09:38,109] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:09:38,112] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:09:38,114] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:09:38,115] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:09:43,921] INFO : VALIDATION | Step=43000/50000 | Train Loss=8.0818 | Valid Loss=7.9589 | Train Loss Unweighted=2.9204 | Valid Loss Unweighted=2.8652 | Stale=0 | ETA=19.1m +rank0 [2026-10-28 13:09:43,930] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:09:43,986] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:09:43,990] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:09:44,169] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:09:45,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:09:46,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:09:46,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:09:47,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:09:47,649] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:09:47,653] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:09:47,707] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:09:47,710] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:09:47,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:09:49,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:09:50,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:09:50,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:09:51,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:09:51,355] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:09:51,358] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:09:51,411] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:09:51,414] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:09:51,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:09:53,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:09:53,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:09:54,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:09:54,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:09:54,997] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:09:55,004] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:09:55,264] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:09:55,675] INFO : Plotted number of elements +rank0 [2026-10-28 13:09:57,120] INFO : Plotted jets +rank0 [2026-10-28 13:09:58,211] INFO : Plotted jet ratio +rank0 [2026-10-28 13:09:58,244] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:09:58,478] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:09:58,872] INFO : Plotted number of elements +rank0 [2026-10-28 13:10:00,841] INFO : Plotted jets +rank0 [2026-10-28 13:10:01,940] INFO : Plotted jet ratio +rank0 [2026-10-28 13:10:01,972] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 13:10:02,181] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 13:10:02,545] INFO : Plotted number of elements +rank0 [2026-10-28 13:10:03,923] INFO : Plotted jets +rank0 [2026-10-28 13:10:04,982] INFO : Plotted jet ratio +rank0 [2026-10-28 13:10:16,224] INFO : Step 43100/50000 rank0 | Train Loss: 7.6511 | Train Loss Unweighted: 2.8176 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5278 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1254 | Regression_eta: 0.0131 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6466 | Classification_binary: 2.5278 | Regression_cos_phi: 0.4357 | Regression_energy: 1.3709 | Regression_eta: 0.5820 | Regression_pt: 1.4320 | Regression_sin_phi: 0.6560 | LR: 1.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3856s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.3324 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6032.3 | uss_MB: 5563.0 | child_rss_MB: 3197.8 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:16,436] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:27,310] INFO : Step 43200/50000 rank0 | Train Loss: 7.9706 | Train Loss Unweighted: 2.8162 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5103 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1329 | Regression_eta: 0.0149 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6748 | Classification_binary: 2.5103 | Regression_cos_phi: 0.4887 | Regression_energy: 1.4524 | Regression_eta: 0.6632 | Regression_pt: 1.4904 | Regression_sin_phi: 0.6908 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4956s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1153 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3542 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6032.3 | uss_MB: 5562.9 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:10:27,523] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:38,330] INFO : Step 43300/50000 rank0 | Train Loss: 7.7773 | Train Loss Unweighted: 2.8255 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5288 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1281 | Regression_eta: 0.0131 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6793 | Classification_binary: 2.5288 | Regression_cos_phi: 0.4785 | Regression_energy: 1.4008 | Regression_eta: 0.5854 | Regression_pt: 1.4656 | Regression_sin_phi: 0.6389 | LR: 1.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4627s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0184 | max_loss: 0.0298 | max_backward: 0.0393 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6032.3 | uss_MB: 5565.0 | child_rss_MB: 3191.6 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:10:38,544] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:49,515] INFO : Step 43400/50000 rank0 | Train Loss: 8.1726 | Train Loss Unweighted: 2.9588 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6436 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1384 | Regression_eta: 0.0120 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7457 | Classification_binary: 2.6436 | Regression_cos_phi: 0.4750 | Regression_energy: 1.5132 | Regression_eta: 0.5372 | Regression_pt: 1.5447 | Regression_sin_phi: 0.7132 | LR: 1.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5157s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1223 | max_loss: 0.0036 | max_backward: 0.0384 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.3751 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5908.3 | uss_MB: 5440.4 | child_rss_MB: 3195.3 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:10:49,727] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:00,592] INFO : Step 43500/50000 rank0 | Train Loss: 8.6365 | Train Loss Unweighted: 2.8841 | Train Loss Components: Classification: 0.0156 | Classification_binary: 2.5406 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1510 | Regression_eta: 0.0250 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.5871 | Classification_binary: 2.5406 | Regression_cos_phi: 0.4640 | Regression_energy: 1.6508 | Regression_eta: 1.1136 | Regression_pt: 1.6396 | Regression_sin_phi: 0.6408 | LR: 1.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0562 | max_loss: 0.0035 | max_backward: 0.0380 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3365 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5909.3 | uss_MB: 5441.6 | child_rss_MB: 3195.5 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:11:00,806] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:11,713] INFO : Step 43600/50000 rank0 | Train Loss: 8.5140 | Train Loss Unweighted: 2.9939 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6513 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1520 | Regression_eta: 0.0174 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6734 | Classification_binary: 2.6513 | Regression_cos_phi: 0.4745 | Regression_energy: 1.6623 | Regression_eta: 0.7769 | Regression_pt: 1.6835 | Regression_sin_phi: 0.5922 | LR: 1.36e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3559s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0549 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5909.3 | uss_MB: 5439.9 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:11:11,930] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:22,760] INFO : Step 43700/50000 rank0 | Train Loss: 7.9316 | Train Loss Unweighted: 2.9018 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5808 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1437 | Regression_eta: 0.0099 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6633 | Classification_binary: 2.5808 | Regression_cos_phi: 0.4141 | Regression_energy: 1.5713 | Regression_eta: 0.4421 | Regression_pt: 1.6236 | Regression_sin_phi: 0.6364 | LR: 1.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4903s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1216 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3531 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5910.3 | uss_MB: 5440.4 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:22,976] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:33,866] INFO : Step 43800/50000 rank0 | Train Loss: 8.1004 | Train Loss Unweighted: 2.8529 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5341 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1408 | Regression_eta: 0.0129 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7080 | Classification_binary: 2.5341 | Regression_cos_phi: 0.4723 | Regression_energy: 1.5390 | Regression_eta: 0.5764 | Regression_pt: 1.5629 | Regression_sin_phi: 0.7078 | LR: 1.34e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3793s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.0344 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3419 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5910.3 | uss_MB: 5440.6 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:34,078] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:45,000] INFO : Step 43900/50000 rank0 | Train Loss: 8.2224 | Train Loss Unweighted: 2.8321 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5034 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1450 | Regression_eta: 0.0153 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6616 | Classification_binary: 2.5034 | Regression_cos_phi: 0.5384 | Regression_energy: 1.5853 | Regression_eta: 0.6811 | Regression_pt: 1.6146 | Regression_sin_phi: 0.6380 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3311s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0227 | max_forward: 0.0144 | max_loss: 0.0036 | max_backward: 0.0338 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5911.3 | uss_MB: 5440.8 | child_rss_MB: 3190.2 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:45,216] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:56,060] INFO : Step 44000/50000 rank0 | Train Loss: 8.3459 | Train Loss Unweighted: 2.8644 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.5309 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1425 | Regression_eta: 0.0230 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6379 | Classification_binary: 2.5309 | Regression_cos_phi: 0.3872 | Regression_energy: 1.5574 | Regression_eta: 1.0275 | Regression_pt: 1.6548 | Regression_sin_phi: 0.5502 | LR: 1.32e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3988s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0223 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5909.5 | uss_MB: 5438.4 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:11:56,276] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:56,295] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 13:11:56,379] INFO : Running validation on rank0 +rank0 [2026-10-28 13:11:56,379] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:11:56,379] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:11:57,484] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 13:11:57,631] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:11:57,764] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:11:57,838] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:11:57,907] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:11:57,975] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:11:58,045] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:11:58,115] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:11:58,226] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:11:58,359] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:11:58,491] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:11:58,563] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:11:58,633] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:11:58,702] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:11:58,772] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:11:58,842] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:11:58,961] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:11:58,968] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:11:58,970] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:11:58,971] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:11:58,974] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:11:58,977] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:11:58,980] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:11:58,982] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:11:58,985] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:11:58,986] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:11:59,096] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:11:59,099] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:11:59,102] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:11:59,104] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:11:59,107] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:11:59,108] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:12:04,949] INFO : VALIDATION | Step=44000/50000 | Train Loss=8.3459 | Valid Loss=7.9725 | Train Loss Unweighted=2.8644 | Valid Loss Unweighted=2.8735 | Stale=1 | ETA=16.3m +rank0 [2026-10-28 13:12:04,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:12:05,014] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:12:05,018] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:12:05,197] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:12:06,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:12:07,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:12:07,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:12:08,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:12:08,663] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:12:08,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:12:08,721] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:12:08,724] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:12:08,846] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:12:10,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:12:10,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:12:11,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:12:12,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:12:12,219] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:12:12,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:12:12,276] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:12:12,278] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:12:12,402] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:12:14,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:12:14,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:12:15,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:12:15,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:12:15,817] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:12:15,825] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:12:16,092] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:12:16,502] INFO : Plotted number of elements +rank0 [2026-10-28 13:12:17,946] INFO : Plotted jets +rank0 [2026-10-28 13:12:19,034] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:19,067] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:12:19,298] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:12:19,695] INFO : Plotted number of elements +rank0 [2026-10-28 13:12:21,087] INFO : Plotted jets +rank0 [2026-10-28 13:12:22,756] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:22,787] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 13:12:22,997] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 13:12:23,361] INFO : Plotted number of elements +rank0 [2026-10-28 13:12:24,744] INFO : Plotted jets +rank0 [2026-10-28 13:12:25,808] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:37,210] INFO : Step 44100/50000 rank0 | Train Loss: 8.4101 | Train Loss Unweighted: 2.9574 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.6243 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1468 | Regression_eta: 0.0179 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6804 | Classification_binary: 2.6243 | Regression_cos_phi: 0.4518 | Regression_energy: 1.6051 | Regression_eta: 0.7961 | Regression_pt: 1.6259 | Regression_sin_phi: 0.6265 | LR: 1.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4925s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1192 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3381 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6097.0 | uss_MB: 5626.9 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:37,421] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:48,281] INFO : Step 44200/50000 rank0 | Train Loss: 8.8956 | Train Loss Unweighted: 2.9259 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5751 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1541 | Regression_eta: 0.0296 | Regression_pt: 0.1355 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7158 | Classification_binary: 2.5751 | Regression_cos_phi: 0.4299 | Regression_energy: 1.6851 | Regression_eta: 1.3217 | Regression_pt: 1.6098 | Regression_sin_phi: 0.5582 | LR: 1.30e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4542s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1063 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.3284 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 6097.0 | uss_MB: 5627.0 | child_rss_MB: 3196.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:48,494] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:59,437] INFO : Step 44300/50000 rank0 | Train Loss: 7.7092 | Train Loss Unweighted: 2.7963 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5001 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1306 | Regression_eta: 0.0123 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7057 | Classification_binary: 2.5001 | Regression_cos_phi: 0.4432 | Regression_energy: 1.4277 | Regression_eta: 0.5483 | Regression_pt: 1.4359 | Regression_sin_phi: 0.6483 | LR: 1.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3530s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0416 | max_forward: 0.0158 | max_loss: 0.0037 | max_backward: 0.0342 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6097.0 | uss_MB: 5627.0 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:59,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:10,427] INFO : Step 44400/50000 rank0 | Train Loss: 7.9654 | Train Loss Unweighted: 2.8240 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5000 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1414 | Regression_eta: 0.0126 | Regression_pt: 0.1388 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6828 | Classification_binary: 2.5000 | Regression_cos_phi: 0.4595 | Regression_energy: 1.5463 | Regression_eta: 0.5630 | Regression_pt: 1.6488 | Regression_sin_phi: 0.5651 | LR: 1.28e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4481s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0036 | max_backward: 0.0338 | max_optimizer_step: 0.1369 | max_tensorboard_logging: 0.3974 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5983.6 | uss_MB: 5513.1 | child_rss_MB: 3193.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:13:10,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:21,623] INFO : Step 44500/50000 rank0 | Train Loss: 8.1268 | Train Loss Unweighted: 2.8547 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5371 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1394 | Regression_eta: 0.0140 | Regression_pt: 0.1302 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7419 | Classification_binary: 2.5371 | Regression_cos_phi: 0.4233 | Regression_energy: 1.5245 | Regression_eta: 0.6256 | Regression_pt: 1.5475 | Regression_sin_phi: 0.7270 | LR: 1.27e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4596s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0845 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3371 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5984.6 | uss_MB: 5514.2 | child_rss_MB: 3190.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:13:21,837] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:32,666] INFO : Step 44600/50000 rank0 | Train Loss: 8.3798 | Train Loss Unweighted: 2.7587 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4114 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1488 | Regression_eta: 0.0188 | Regression_pt: 0.1478 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7230 | Classification_binary: 2.4114 | Regression_cos_phi: 0.3937 | Regression_energy: 1.6270 | Regression_eta: 0.8383 | Regression_pt: 1.7558 | Regression_sin_phi: 0.6306 | LR: 1.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0172 | max_backward: 0.0338 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5984.6 | uss_MB: 5514.7 | child_rss_MB: 3192.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:13:32,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:43,865] INFO : Step 44700/50000 rank0 | Train Loss: 8.0817 | Train Loss Unweighted: 2.8372 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5158 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1404 | Regression_eta: 0.0150 | Regression_pt: 0.1343 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6768 | Classification_binary: 2.5158 | Regression_cos_phi: 0.4557 | Regression_energy: 1.5349 | Regression_eta: 0.6682 | Regression_pt: 1.5962 | Regression_sin_phi: 0.6340 | LR: 1.25e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3883s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0292 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.3337 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5984.6 | uss_MB: 5514.8 | child_rss_MB: 3195.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:13:44,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:55,010] INFO : Step 44800/50000 rank0 | Train Loss: 7.7122 | Train Loss Unweighted: 2.8065 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4914 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1401 | Regression_eta: 0.0106 | Regression_pt: 0.1343 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6981 | Classification_binary: 2.4914 | Regression_cos_phi: 0.3535 | Regression_energy: 1.5321 | Regression_eta: 0.4727 | Regression_pt: 1.5957 | Regression_sin_phi: 0.5687 | LR: 1.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4387s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0019 | max_forward: 0.0719 | max_loss: 0.0035 | max_backward: 0.0387 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.3477 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5984.6 | uss_MB: 5517.3 | child_rss_MB: 3196.0 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:13:55,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:06,027] INFO : Step 44900/50000 rank0 | Train Loss: 8.1321 | Train Loss Unweighted: 2.8843 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5749 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1337 | Regression_eta: 0.0122 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6784 | Classification_binary: 2.5749 | Regression_cos_phi: 0.6306 | Regression_energy: 1.4619 | Regression_eta: 0.5427 | Regression_pt: 1.5222 | Regression_sin_phi: 0.7214 | LR: 1.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4607s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0015 | max_forward: 0.0144 | max_loss: 0.0536 | max_backward: 0.0334 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.3813 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5984.6 | uss_MB: 5515.2 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:06,238] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:17,172] INFO : Step 45000/50000 rank0 | Train Loss: 7.8200 | Train Loss Unweighted: 2.9315 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.6181 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1375 | Regression_eta: 0.0099 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6491 | Classification_binary: 2.6181 | Regression_cos_phi: 0.4011 | Regression_energy: 1.5034 | Regression_eta: 0.4400 | Regression_pt: 1.6210 | Regression_sin_phi: 0.5874 | LR: 1.22e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4616s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0267 | max_backward: 0.0335 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.3779 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5984.6 | uss_MB: 5515.2 | child_rss_MB: 3192.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:17,384] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:17,498] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 13:14:17,582] INFO : Running validation on rank0 +rank0 [2026-10-28 13:14:17,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:14:17,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:14:18,686] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 13:14:18,834] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:14:18,967] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:14:19,040] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:14:19,109] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:14:19,178] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:14:19,248] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:14:19,317] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:14:19,428] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:14:19,562] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:14:19,695] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:14:19,767] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:14:19,836] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:14:19,906] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:14:19,976] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:14:20,047] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:14:20,155] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:14:20,162] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:14:20,165] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:14:20,168] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:14:20,171] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:14:20,172] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:14:20,174] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:14:20,177] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:14:20,178] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:14:20,181] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:14:20,183] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:14:20,186] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:14:20,188] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:14:20,190] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:14:20,296] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:14:20,299] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:14:26,162] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.8200 | Valid Loss=7.9041 | Train Loss Unweighted=2.9315 | Valid Loss Unweighted=2.8544 | Stale=0 | ETA=13.5m +rank0 [2026-10-28 13:14:26,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:26,228] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:14:26,231] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:14:26,412] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:14:27,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:14:28,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:14:29,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:14:29,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:14:29,907] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:14:29,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:29,966] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:14:29,969] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:14:30,085] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:14:31,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:14:32,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:14:32,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:14:33,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:14:33,701] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:14:33,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:33,758] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:14:33,761] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:14:33,890] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:14:35,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:14:35,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:14:36,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:14:37,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:14:37,236] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:14:37,243] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:14:37,506] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:14:37,917] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:39,361] INFO : Plotted jets +rank0 [2026-10-28 13:14:40,459] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:40,491] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:14:40,730] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:14:41,121] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:42,516] INFO : Plotted jets +rank0 [2026-10-28 13:14:44,185] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:44,218] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 13:14:44,428] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 13:14:44,792] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:46,174] INFO : Plotted jets +rank0 [2026-10-28 13:14:47,249] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:58,628] INFO : Step 45100/50000 rank0 | Train Loss: 8.0934 | Train Loss Unweighted: 2.7238 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4080 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1364 | Regression_eta: 0.0156 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7362 | Classification_binary: 2.4080 | Regression_cos_phi: 0.5595 | Regression_energy: 1.4911 | Regression_eta: 0.6970 | Regression_pt: 1.5267 | Regression_sin_phi: 0.6750 | LR: 1.21e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4586s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0169 | max_loss: 0.0035 | max_backward: 0.0383 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6172.3 | uss_MB: 5705.3 | child_rss_MB: 3188.5 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 13:14:58,842] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:09,808] INFO : Step 45200/50000 rank0 | Train Loss: 8.4949 | Train Loss Unweighted: 2.9645 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.6296 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1481 | Regression_eta: 0.0161 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6654 | Classification_binary: 2.6296 | Regression_cos_phi: 0.4551 | Regression_energy: 1.6190 | Regression_eta: 0.7175 | Regression_pt: 1.6377 | Regression_sin_phi: 0.7706 | LR: 1.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3939s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0035 | max_backward: 0.0340 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3329 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6172.3 | uss_MB: 5703.3 | child_rss_MB: 3194.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:15:10,022] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:20,918] INFO : Step 45300/50000 rank0 | Train Loss: 7.9847 | Train Loss Unweighted: 2.9184 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.5979 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1430 | Regression_eta: 0.0077 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7007 | Classification_binary: 2.5979 | Regression_cos_phi: 0.4674 | Regression_energy: 1.5630 | Regression_eta: 0.3431 | Regression_pt: 1.6238 | Regression_sin_phi: 0.6887 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4848s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1019 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1383 | max_tensorboard_logging: 0.3377 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6172.3 | uss_MB: 5703.3 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:21,131] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:32,057] INFO : Step 45400/50000 rank0 | Train Loss: 7.3751 | Train Loss Unweighted: 2.7944 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5013 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1303 | Regression_eta: 0.0084 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6398 | Classification_binary: 2.5013 | Regression_cos_phi: 0.4017 | Regression_energy: 1.4240 | Regression_eta: 0.3732 | Regression_pt: 1.4920 | Regression_sin_phi: 0.5430 | LR: 1.19e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1126 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3384 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6172.3 | uss_MB: 5703.3 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:32,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:43,054] INFO : Step 45500/50000 rank0 | Train Loss: 7.3840 | Train Loss Unweighted: 2.7373 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4482 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1248 | Regression_eta: 0.0089 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6845 | Classification_binary: 2.4482 | Regression_cos_phi: 0.3835 | Regression_energy: 1.3640 | Regression_eta: 0.3979 | Regression_pt: 1.4821 | Regression_sin_phi: 0.6237 | LR: 1.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4645s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0052 | max_backward: 0.0331 | max_optimizer_step: 0.1368 | max_tensorboard_logging: 0.3982 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6053.2 | uss_MB: 5585.0 | child_rss_MB: 3188.8 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:15:43,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:54,222] INFO : Step 45600/50000 rank0 | Train Loss: 8.1330 | Train Loss Unweighted: 2.7890 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4483 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1510 | Regression_eta: 0.0120 | Regression_pt: 0.1452 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7292 | Classification_binary: 2.4483 | Regression_cos_phi: 0.4262 | Regression_energy: 1.6512 | Regression_eta: 0.5335 | Regression_pt: 1.7258 | Regression_sin_phi: 0.6189 | LR: 1.17e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4844s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1015 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1367 | max_tensorboard_logging: 0.3435 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6053.2 | uss_MB: 5585.2 | child_rss_MB: 3192.2 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:54,438] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:05,311] INFO : Step 45700/50000 rank0 | Train Loss: 8.3422 | Train Loss Unweighted: 2.8696 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5177 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1582 | Regression_eta: 0.0120 | Regression_pt: 0.1498 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6701 | Classification_binary: 2.5177 | Regression_cos_phi: 0.4934 | Regression_energy: 1.7293 | Regression_eta: 0.5365 | Regression_pt: 1.7797 | Regression_sin_phi: 0.6155 | LR: 1.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3842s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3227 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6053.2 | uss_MB: 5585.2 | child_rss_MB: 3187.8 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:05,525] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:16,449] INFO : Step 45800/50000 rank0 | Train Loss: 8.5766 | Train Loss Unweighted: 2.8694 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5458 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1378 | Regression_eta: 0.0165 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7052 | Classification_binary: 2.5458 | Regression_cos_phi: 0.6524 | Regression_energy: 1.5067 | Regression_eta: 0.7351 | Regression_pt: 1.5610 | Regression_sin_phi: 0.8703 | LR: 1.16e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4250s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1034 | max_forward: 0.0139 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6053.2 | uss_MB: 5585.3 | child_rss_MB: 3190.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:16:16,664] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:27,477] INFO : Step 45900/50000 rank0 | Train Loss: 8.3949 | Train Loss Unweighted: 2.9452 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.6034 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1508 | Regression_eta: 0.0115 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7076 | Classification_binary: 2.6034 | Regression_cos_phi: 0.4580 | Regression_energy: 1.6487 | Regression_eta: 0.5137 | Regression_pt: 1.7336 | Regression_sin_phi: 0.7299 | LR: 1.15e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4893s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1212 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3471 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6054.2 | uss_MB: 5585.8 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:27,693] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:38,604] INFO : Step 46000/50000 rank0 | Train Loss: 8.3493 | Train Loss Unweighted: 2.8210 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4917 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1439 | Regression_eta: 0.0185 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6737 | Classification_binary: 2.4917 | Regression_cos_phi: 0.5224 | Regression_energy: 1.5731 | Regression_eta: 0.8266 | Regression_pt: 1.5902 | Regression_sin_phi: 0.6717 | LR: 1.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4054s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0368 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.3353 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6054.2 | uss_MB: 5585.8 | child_rss_MB: 3190.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:38,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:38,846] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 13:16:38,930] INFO : Running validation on rank0 +rank0 [2026-10-28 13:16:38,930] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:16:38,930] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:16:40,030] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 13:16:40,177] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:16:40,313] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:16:40,388] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:16:40,458] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:16:40,526] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:16:40,596] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:16:40,667] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:16:40,778] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:16:40,912] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:16:41,044] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:16:41,116] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:16:41,185] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:16:41,254] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:16:41,324] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:16:41,395] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:16:41,511] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:16:41,518] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:16:41,521] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:16:41,524] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:16:41,525] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:16:41,528] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:16:41,636] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:16:41,639] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:16:41,642] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:16:41,643] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:16:41,645] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:16:41,648] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:16:41,649] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:16:41,652] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:16:41,655] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:16:41,658] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:16:47,584] INFO : VALIDATION | Step=46000/50000 | Train Loss=8.3493 | Valid Loss=7.9248 | Train Loss Unweighted=2.8210 | Valid Loss Unweighted=2.8603 | Stale=1 | ETA=10.8m +rank0 [2026-10-28 13:16:47,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:16:47,648] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:16:47,651] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:16:47,834] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:16:49,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:16:50,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:16:50,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:16:51,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:16:51,512] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:16:51,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:16:51,570] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:16:51,573] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:16:51,699] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:16:53,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:16:53,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:16:54,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:16:55,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:16:55,254] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:16:55,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:16:55,311] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:16:55,314] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:16:55,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:16:56,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:16:57,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:16:58,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:16:58,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:16:58,668] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:16:58,675] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:16:58,936] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:16:59,345] INFO : Plotted number of elements +rank0 [2026-10-28 13:17:00,796] INFO : Plotted jets +rank0 [2026-10-28 13:17:01,896] INFO : Plotted jet ratio +rank0 [2026-10-28 13:17:01,928] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:17:02,158] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:17:02,553] INFO : Plotted number of elements +rank0 [2026-10-28 13:17:03,954] INFO : Plotted jets +rank0 [2026-10-28 13:17:05,637] INFO : Plotted jet ratio +rank0 [2026-10-28 13:17:05,669] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 13:17:05,874] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 13:17:06,238] INFO : Plotted number of elements +rank0 [2026-10-28 13:17:07,624] INFO : Plotted jets +rank0 [2026-10-28 13:17:08,699] INFO : Plotted jet ratio +rank0 [2026-10-28 13:17:20,145] INFO : Step 46100/50000 rank0 | Train Loss: 7.9236 | Train Loss Unweighted: 2.8441 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5189 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1440 | Regression_eta: 0.0127 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6982 | Classification_binary: 2.5189 | Regression_cos_phi: 0.3615 | Regression_energy: 1.5744 | Regression_eta: 0.5675 | Regression_pt: 1.6446 | Regression_sin_phi: 0.5585 | LR: 1.13e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0542 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1369 | max_tensorboard_logging: 0.3316 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6249.5 | uss_MB: 5780.2 | child_rss_MB: 3189.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:20,356] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:31,485] INFO : Step 46200/50000 rank0 | Train Loss: 7.5671 | Train Loss Unweighted: 2.8368 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.5311 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1347 | Regression_eta: 0.0092 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6503 | Classification_binary: 2.5311 | Regression_cos_phi: 0.3654 | Regression_energy: 1.4729 | Regression_eta: 0.4105 | Regression_pt: 1.5800 | Regression_sin_phi: 0.5568 | LR: 1.13e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3593s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1536 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6249.5 | uss_MB: 5780.2 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:17:31,697] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:42,420] INFO : Step 46300/50000 rank0 | Train Loss: 7.8275 | Train Loss Unweighted: 2.8179 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.4919 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1452 | Regression_eta: 0.0088 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6264 | Classification_binary: 2.4919 | Regression_cos_phi: 0.4099 | Regression_energy: 1.5877 | Regression_eta: 0.3921 | Regression_pt: 1.6925 | Regression_sin_phi: 0.6270 | LR: 1.12e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3552s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6249.5 | uss_MB: 5780.2 | child_rss_MB: 3192.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:42,640] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:53,518] INFO : Step 46400/50000 rank0 | Train Loss: 7.7274 | Train Loss Unweighted: 2.8074 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.5025 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1355 | Regression_eta: 0.0100 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7227 | Classification_binary: 2.5025 | Regression_cos_phi: 0.4556 | Regression_energy: 1.4813 | Regression_eta: 0.4470 | Regression_pt: 1.5060 | Regression_sin_phi: 0.6123 | LR: 1.11e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4427s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1010 | max_forward: 0.0159 | max_loss: 0.0034 | max_backward: 0.0382 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6249.5 | uss_MB: 5782.3 | child_rss_MB: 3193.8 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:17:53,734] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:04,536] INFO : Step 46500/50000 rank0 | Train Loss: 8.3743 | Train Loss Unweighted: 2.9927 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6670 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1406 | Regression_eta: 0.0145 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7244 | Classification_binary: 2.6670 | Regression_cos_phi: 0.4850 | Regression_energy: 1.5372 | Regression_eta: 0.6486 | Regression_pt: 1.6225 | Regression_sin_phi: 0.6895 | LR: 1.11e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3809s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.3250 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6128.5 | uss_MB: 5659.8 | child_rss_MB: 3194.2 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:18:04,751] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:15,654] INFO : Step 46600/50000 rank0 | Train Loss: 7.3867 | Train Loss Unweighted: 2.7142 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4266 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1272 | Regression_eta: 0.0092 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6901 | Classification_binary: 2.4266 | Regression_cos_phi: 0.3956 | Regression_energy: 1.3901 | Regression_eta: 0.4101 | Regression_pt: 1.4255 | Regression_sin_phi: 0.6487 | LR: 1.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4850s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1208 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1352 | max_tensorboard_logging: 0.3378 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6129.5 | uss_MB: 5660.3 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:18:15,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:26,797] INFO : Step 46700/50000 rank0 | Train Loss: 8.6452 | Train Loss Unweighted: 3.0321 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6801 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1536 | Regression_eta: 0.0127 | Regression_pt: 0.1512 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7399 | Classification_binary: 2.6801 | Regression_cos_phi: 0.5023 | Regression_energy: 1.6788 | Regression_eta: 0.5645 | Regression_pt: 1.7961 | Regression_sin_phi: 0.6834 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3430s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0271 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6129.5 | uss_MB: 5660.4 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:18:27,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:37,846] INFO : Step 46800/50000 rank0 | Train Loss: 7.5495 | Train Loss Unweighted: 2.7842 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5016 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1206 | Regression_eta: 0.0136 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6614 | Classification_binary: 2.5016 | Regression_cos_phi: 0.4530 | Regression_energy: 1.3190 | Regression_eta: 0.6067 | Regression_pt: 1.3939 | Regression_sin_phi: 0.6139 | LR: 1.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4431s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0676 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3294 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6129.5 | uss_MB: 5660.7 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:18:38,060] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:48,904] INFO : Step 46900/50000 rank0 | Train Loss: 7.7648 | Train Loss Unweighted: 2.8401 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5413 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1299 | Regression_eta: 0.0121 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7104 | Classification_binary: 2.5413 | Regression_cos_phi: 0.4596 | Regression_energy: 1.4204 | Regression_eta: 0.5419 | Regression_pt: 1.4765 | Regression_sin_phi: 0.6146 | LR: 1.09e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0172 | max_loss: 0.0594 | max_backward: 0.0387 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.3876 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6129.5 | uss_MB: 5663.2 | child_rss_MB: 3197.9 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:18:49,117] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:00,083] INFO : Step 47000/50000 rank0 | Train Loss: 7.5365 | Train Loss Unweighted: 2.8532 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5705 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1209 | Regression_eta: 0.0103 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7614 | Classification_binary: 2.5705 | Regression_cos_phi: 0.4080 | Regression_energy: 1.3218 | Regression_eta: 0.4590 | Regression_pt: 1.4083 | Regression_sin_phi: 0.6074 | LR: 1.08e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4671s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1097 | max_loss: 0.0036 | max_backward: 0.0351 | max_optimizer_step: 0.1370 | max_tensorboard_logging: 0.3305 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6129.5 | uss_MB: 5661.5 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:19:00,296] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:00,333] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 13:19:00,425] INFO : Running validation on rank0 +rank0 [2026-10-28 13:19:00,426] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:19:00,426] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:19:01,528] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 13:19:01,676] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:19:01,809] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:19:01,883] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:19:01,966] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:19:02,040] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:19:02,110] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:19:02,180] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:19:02,307] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:19:02,441] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:19:02,574] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:19:02,646] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:19:02,716] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:19:02,786] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:19:02,857] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:19:02,927] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:19:03,045] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:19:03,052] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:19:03,055] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:19:03,058] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:19:03,059] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:19:03,062] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:19:03,063] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:19:03,066] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:19:03,168] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:19:03,171] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:19:03,174] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:19:03,177] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:19:03,178] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:19:03,180] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:19:03,183] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:19:03,184] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:19:09,037] INFO : VALIDATION | Step=47000/50000 | Train Loss=7.5365 | Valid Loss=7.9068 | Train Loss Unweighted=2.8532 | Valid Loss Unweighted=2.8517 | Stale=2 | ETA=8.1m +rank0 [2026-10-28 13:19:09,046] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:19:09,103] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:19:09,107] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:19:09,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:19:10,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:19:11,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:19:12,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:19:12,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:19:12,757] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:19:12,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:19:12,817] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:19:12,820] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:12,942] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:19:14,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:19:15,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:19:15,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:19:16,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:19:16,460] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:19:16,463] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:19:16,517] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:19:16,520] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:19:16,641] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:19:18,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:19:18,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:19:19,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:19:19,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:19:20,118] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:19:20,126] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:19:20,385] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:19:20,796] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:22,240] INFO : Plotted jets +rank0 [2026-10-28 13:19:23,324] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:23,356] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:23,589] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:23,982] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:25,374] INFO : Plotted jets +rank0 [2026-10-28 13:19:27,051] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:27,084] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:27,294] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:27,659] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:29,033] INFO : Plotted jets +rank0 [2026-10-28 13:19:30,097] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:41,534] INFO : Step 47100/50000 rank0 | Train Loss: 7.8766 | Train Loss Unweighted: 2.8719 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.5630 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1355 | Regression_eta: 0.0101 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6588 | Classification_binary: 2.5630 | Regression_cos_phi: 0.5587 | Regression_energy: 1.4811 | Regression_eta: 0.4494 | Regression_pt: 1.5535 | Regression_sin_phi: 0.6121 | LR: 1.07e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1224 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3242 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6321.9 | uss_MB: 5851.5 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:19:41,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:52,588] INFO : Step 47200/50000 rank0 | Train Loss: 8.0834 | Train Loss Unweighted: 2.8372 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5106 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1445 | Regression_eta: 0.0145 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6882 | Classification_binary: 2.5106 | Regression_cos_phi: 0.3891 | Regression_energy: 1.5796 | Regression_eta: 0.6453 | Regression_pt: 1.6229 | Regression_sin_phi: 0.6477 | LR: 1.07e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4661s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0095 | max_backward: 0.0330 | max_optimizer_step: 0.1414 | max_tensorboard_logging: 0.3801 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6321.9 | uss_MB: 5851.5 | child_rss_MB: 3196.4 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:19:52,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:03,782] INFO : Step 47300/50000 rank0 | Train Loss: 8.7199 | Train Loss Unweighted: 2.8333 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4913 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1458 | Regression_eta: 0.0235 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7360 | Classification_binary: 2.4913 | Regression_cos_phi: 0.4967 | Regression_energy: 1.5940 | Regression_eta: 1.0479 | Regression_pt: 1.6402 | Regression_sin_phi: 0.7138 | LR: 1.06e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4249s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0377 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.3201 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6321.9 | uss_MB: 5851.6 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:03,997] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:14,988] INFO : Step 47400/50000 rank0 | Train Loss: 7.6852 | Train Loss Unweighted: 2.7405 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4170 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1445 | Regression_eta: 0.0085 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6719 | Classification_binary: 2.4170 | Regression_cos_phi: 0.3571 | Regression_energy: 1.5798 | Regression_eta: 0.3777 | Regression_pt: 1.6727 | Regression_sin_phi: 0.6089 | LR: 1.06e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4519s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0903 | max_forward: 0.0169 | max_loss: 0.0036 | max_backward: 0.0382 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6321.9 | uss_MB: 5853.6 | child_rss_MB: 3193.0 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:20:15,209] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:26,093] INFO : Step 47500/50000 rank0 | Train Loss: 8.3020 | Train Loss Unweighted: 2.9092 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.5762 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1459 | Regression_eta: 0.0105 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6603 | Classification_binary: 2.5762 | Regression_cos_phi: 0.4747 | Regression_energy: 1.5955 | Regression_eta: 0.4689 | Regression_pt: 1.6988 | Regression_sin_phi: 0.8276 | LR: 1.06e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0295 | max_forward: 0.0141 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3108 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6203.7 | uss_MB: 5733.0 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:26,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:37,162] INFO : Step 47600/50000 rank0 | Train Loss: 7.6789 | Train Loss Unweighted: 2.7988 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4885 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1358 | Regression_eta: 0.0082 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6744 | Classification_binary: 2.4885 | Regression_cos_phi: 0.4495 | Regression_energy: 1.4850 | Regression_eta: 0.3673 | Regression_pt: 1.6033 | Regression_sin_phi: 0.6110 | LR: 1.05e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4323s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0913 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6204.7 | uss_MB: 5733.9 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:37,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:48,227] INFO : Step 47700/50000 rank0 | Train Loss: 8.0356 | Train Loss Unweighted: 2.7686 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.4499 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1422 | Regression_eta: 0.0170 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6506 | Classification_binary: 2.4499 | Regression_cos_phi: 0.4911 | Regression_energy: 1.5549 | Regression_eta: 0.7572 | Regression_pt: 1.5236 | Regression_sin_phi: 0.6084 | LR: 1.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3145s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0036 | max_backward: 0.0333 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.2677 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6204.7 | uss_MB: 5733.9 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:20:48,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:59,342] INFO : Step 47800/50000 rank0 | Train Loss: 8.5156 | Train Loss Unweighted: 2.8458 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5181 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1402 | Regression_eta: 0.0257 | Regression_pt: 0.1299 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6962 | Classification_binary: 2.5181 | Regression_cos_phi: 0.4170 | Regression_energy: 1.5328 | Regression_eta: 1.1460 | Regression_pt: 1.5434 | Regression_sin_phi: 0.6621 | LR: 1.04e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0982 | max_forward: 0.0148 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6205.7 | uss_MB: 5734.4 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:59,559] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:10,373] INFO : Step 47900/50000 rank0 | Train Loss: 7.7110 | Train Loss Unweighted: 2.7371 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4359 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1301 | Regression_eta: 0.0129 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6803 | Classification_binary: 2.4359 | Regression_cos_phi: 0.4427 | Regression_energy: 1.4221 | Regression_eta: 0.5769 | Regression_pt: 1.5012 | Regression_sin_phi: 0.6518 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3724s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.3012 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6206.7 | uss_MB: 5734.7 | child_rss_MB: 3194.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:21:10,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:21,463] INFO : Step 48000/50000 rank0 | Train Loss: 7.4383 | Train Loss Unweighted: 2.7485 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4632 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1235 | Regression_eta: 0.0099 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7051 | Classification_binary: 2.4632 | Regression_cos_phi: 0.4259 | Regression_energy: 1.3502 | Regression_eta: 0.4414 | Regression_pt: 1.4264 | Regression_sin_phi: 0.6261 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1059 | max_loss: 0.0035 | max_backward: 0.0345 | max_optimizer_step: 0.1414 | max_tensorboard_logging: 0.3000 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 6206.7 | uss_MB: 5735.2 | child_rss_MB: 3193.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:21,675] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:21,730] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 13:21:21,814] INFO : Running validation on rank0 +rank0 [2026-10-28 13:21:21,814] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:21:21,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:21:22,908] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 13:21:23,055] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:21:23,188] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:21:23,263] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:21:23,333] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:21:23,401] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:21:23,470] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:21:23,541] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:21:23,652] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:21:23,786] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:21:23,919] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:21:23,991] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:21:24,061] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:21:24,130] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:21:24,200] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:21:24,270] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:21:24,389] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:21:24,397] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:21:24,400] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:21:24,402] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:21:24,403] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:21:24,406] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:21:24,409] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:21:24,412] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:21:24,415] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:21:24,416] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:21:24,419] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:21:24,421] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:21:24,422] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:21:24,425] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:21:24,426] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:21:24,428] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:21:30,246] INFO : VALIDATION | Step=48000/50000 | Train Loss=7.4383 | Valid Loss=7.8314 | Train Loss Unweighted=2.7485 | Valid Loss Unweighted=2.8333 | Stale=0 | ETA=5.4m +rank0 [2026-10-28 13:21:30,255] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:21:30,312] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:21:30,316] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:21:30,499] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:21:32,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:21:32,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:21:33,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:21:34,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:21:34,355] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:21:34,358] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:21:34,414] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:21:34,417] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:21:34,542] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:21:36,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:21:36,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:21:37,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:21:37,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:21:38,096] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:21:38,100] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:21:38,152] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:21:38,155] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:21:38,280] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:21:39,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:21:40,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:21:41,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:21:41,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:21:41,831] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:21:41,839] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:21:42,101] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:21:42,511] INFO : Plotted number of elements +rank0 [2026-10-28 13:21:43,954] INFO : Plotted jets +rank0 [2026-10-28 13:21:45,051] INFO : Plotted jet ratio +rank0 [2026-10-28 13:21:45,083] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:21:45,318] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:21:45,712] INFO : Plotted number of elements +rank0 [2026-10-28 13:21:47,689] INFO : Plotted jets +rank0 [2026-10-28 13:21:48,804] INFO : Plotted jet ratio +rank0 [2026-10-28 13:21:48,837] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 13:21:49,048] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 13:21:49,411] INFO : Plotted number of elements +rank0 [2026-10-28 13:21:50,799] INFO : Plotted jets +rank0 [2026-10-28 13:21:51,864] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:03,362] INFO : Step 48100/50000 rank0 | Train Loss: 7.9457 | Train Loss Unweighted: 2.8769 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5763 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1304 | Regression_eta: 0.0133 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6980 | Classification_binary: 2.5763 | Regression_cos_phi: 0.5065 | Regression_energy: 1.4254 | Regression_eta: 0.5923 | Regression_pt: 1.4655 | Regression_sin_phi: 0.6817 | LR: 1.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4609s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1072 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0386 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2730 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6396.8 | uss_MB: 5926.1 | child_rss_MB: 3192.2 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:22:03,579] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:14,433] INFO : Step 48200/50000 rank0 | Train Loss: 7.8010 | Train Loss Unweighted: 2.7834 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4778 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1337 | Regression_eta: 0.0112 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6997 | Classification_binary: 2.4778 | Regression_cos_phi: 0.4613 | Regression_energy: 1.4612 | Regression_eta: 0.5008 | Regression_pt: 1.5193 | Regression_sin_phi: 0.6809 | LR: 1.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3733s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0166 | max_forward: 0.0133 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6396.8 | uss_MB: 5924.0 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:14,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:25,424] INFO : Step 48300/50000 rank0 | Train Loss: 7.9939 | Train Loss Unweighted: 2.8113 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4856 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1428 | Regression_eta: 0.0102 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7513 | Classification_binary: 2.4856 | Regression_cos_phi: 0.4222 | Regression_energy: 1.5609 | Regression_eta: 0.4553 | Regression_pt: 1.6545 | Regression_sin_phi: 0.6642 | LR: 1.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4352s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0612 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.3156 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6396.8 | uss_MB: 5924.0 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:22:25,636] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:36,624] INFO : Step 48400/50000 rank0 | Train Loss: 8.5333 | Train Loss Unweighted: 2.9045 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5579 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1531 | Regression_eta: 0.0096 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8004 | Classification_binary: 2.5579 | Regression_cos_phi: 0.6014 | Regression_energy: 1.6742 | Regression_eta: 0.4276 | Regression_pt: 1.7294 | Regression_sin_phi: 0.7425 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4287s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0755 | max_forward: 0.0150 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6396.8 | uss_MB: 5924.0 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:36,837] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:47,584] INFO : Step 48500/50000 rank0 | Train Loss: 7.7259 | Train Loss Unweighted: 2.8036 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5018 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1337 | Regression_eta: 0.0140 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6815 | Classification_binary: 2.5018 | Regression_cos_phi: 0.4428 | Regression_energy: 1.4620 | Regression_eta: 0.6243 | Regression_pt: 1.4654 | Regression_sin_phi: 0.5480 | LR: 1.02e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0133 | max_loss: 0.0354 | max_backward: 0.0330 | max_optimizer_step: 0.1403 | max_tensorboard_logging: 0.3999 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6278.9 | uss_MB: 5806.0 | child_rss_MB: 3194.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:22:47,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:58,744] INFO : Step 48600/50000 rank0 | Train Loss: 7.5326 | Train Loss Unweighted: 2.7159 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.4124 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1334 | Regression_eta: 0.0083 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7980 | Classification_binary: 2.4124 | Regression_cos_phi: 0.4076 | Regression_energy: 1.4582 | Regression_eta: 0.3683 | Regression_pt: 1.5268 | Regression_sin_phi: 0.5614 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4455s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0839 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3287 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6279.9 | uss_MB: 5807.0 | child_rss_MB: 3193.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:58,957] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:09,898] INFO : Step 48700/50000 rank0 | Train Loss: 8.0103 | Train Loss Unweighted: 2.8529 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5429 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1360 | Regression_eta: 0.0136 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7146 | Classification_binary: 2.5429 | Regression_cos_phi: 0.4032 | Regression_energy: 1.4867 | Regression_eta: 0.6062 | Regression_pt: 1.5135 | Regression_sin_phi: 0.7432 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4631s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0582 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3601 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6280.9 | uss_MB: 5807.2 | child_rss_MB: 3191.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:10,113] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:20,898] INFO : Step 48800/50000 rank0 | Train Loss: 7.3196 | Train Loss Unweighted: 2.6804 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3973 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1236 | Regression_eta: 0.0074 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6393 | Classification_binary: 2.3973 | Regression_cos_phi: 0.4885 | Regression_energy: 1.3513 | Regression_eta: 0.3319 | Regression_pt: 1.4299 | Regression_sin_phi: 0.6814 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4574s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0161 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 0.3797 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6280.9 | uss_MB: 5809.9 | child_rss_MB: 3196.5 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:23:21,111] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:32,043] INFO : Step 48900/50000 rank0 | Train Loss: 7.8801 | Train Loss Unweighted: 2.8892 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5804 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1369 | Regression_eta: 0.0105 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7035 | Classification_binary: 2.5804 | Regression_cos_phi: 0.5000 | Regression_energy: 1.4961 | Regression_eta: 0.4661 | Regression_pt: 1.5297 | Regression_sin_phi: 0.6043 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.3010 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 6280.9 | uss_MB: 5807.9 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:32,256] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:43,151] INFO : Step 49000/50000 rank0 | Train Loss: 7.4365 | Train Loss Unweighted: 2.7296 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4395 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1267 | Regression_eta: 0.0102 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6775 | Classification_binary: 2.4395 | Regression_cos_phi: 0.4473 | Regression_energy: 1.3851 | Regression_eta: 0.4551 | Regression_pt: 1.4518 | Regression_sin_phi: 0.5801 | LR: 1.01e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1104 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3170 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6280.9 | uss_MB: 5807.9 | child_rss_MB: 3193.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:43,365] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:43,398] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 13:23:43,481] INFO : Running validation on rank0 +rank0 [2026-10-28 13:23:43,481] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:23:43,481] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:23:44,598] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 13:23:44,745] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:23:44,879] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:23:44,954] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:23:45,024] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:23:45,092] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:23:45,162] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:23:45,232] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:23:45,343] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:23:45,477] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:23:45,610] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:23:45,682] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:23:45,751] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:23:45,820] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:23:45,890] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:23:45,960] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:23:46,076] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:23:46,084] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:23:46,086] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:23:46,192] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:23:46,195] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:23:46,198] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:23:46,201] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:23:46,202] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:23:46,204] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:23:46,207] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:23:46,208] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:23:46,210] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:23:46,213] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:23:46,214] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:23:46,216] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:23:46,219] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:23:52,175] INFO : VALIDATION | Step=49000/50000 | Train Loss=7.4365 | Valid Loss=7.8849 | Train Loss Unweighted=2.7296 | Valid Loss Unweighted=2.8501 | Stale=1 | ETA=2.7m +rank0 [2026-10-28 13:23:52,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:23:52,241] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:23:52,244] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:23:52,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:23:54,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:23:54,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:23:55,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:23:55,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:23:56,032] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:23:56,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:23:56,090] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:23:56,093] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:23:56,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:23:57,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:23:58,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:23:59,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:23:59,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:23:59,850] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:23:59,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:23:59,908] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:23:59,911] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:24:00,041] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:24:01,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:24:02,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:24:02,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:24:03,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:24:03,301] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:24:03,309] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:24:03,571] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:24:03,984] INFO : Plotted number of elements +rank0 [2026-10-28 13:24:05,435] INFO : Plotted jets +rank0 [2026-10-28 13:24:06,522] INFO : Plotted jet ratio +rank0 [2026-10-28 13:24:06,554] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:24:06,788] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:24:07,183] INFO : Plotted number of elements +rank0 [2026-10-28 13:24:08,571] INFO : Plotted jets +rank0 [2026-10-28 13:24:10,256] INFO : Plotted jet ratio +rank0 [2026-10-28 13:24:10,289] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 13:24:10,498] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 13:24:10,860] INFO : Plotted number of elements +rank0 [2026-10-28 13:24:12,242] INFO : Plotted jets +rank0 [2026-10-28 13:24:13,310] INFO : Plotted jet ratio +rank0 [2026-10-28 13:24:24,769] INFO : Step 49100/50000 rank0 | Train Loss: 7.7791 | Train Loss Unweighted: 2.8700 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5803 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1252 | Regression_eta: 0.0138 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6637 | Classification_binary: 2.5803 | Regression_cos_phi: 0.4740 | Regression_energy: 1.3686 | Regression_eta: 0.6141 | Regression_pt: 1.4115 | Regression_sin_phi: 0.6669 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5036s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1172 | max_loss: 0.0036 | max_backward: 0.0334 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3029 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6467.5 | uss_MB: 5997.7 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:24:24,981] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:35,929] INFO : Step 49200/50000 rank0 | Train Loss: 7.6447 | Train Loss Unweighted: 2.7896 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4989 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1273 | Regression_eta: 0.0119 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6280 | Classification_binary: 2.4989 | Regression_cos_phi: 0.5413 | Regression_energy: 1.3921 | Regression_eta: 0.5303 | Regression_pt: 1.4238 | Regression_sin_phi: 0.6303 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0541 | max_forward: 0.0141 | max_loss: 0.0036 | max_backward: 0.0330 | max_optimizer_step: 0.1411 | max_tensorboard_logging: 0.2631 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6467.5 | uss_MB: 5997.7 | child_rss_MB: 3186.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:24:36,145] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:47,051] INFO : Step 49300/50000 rank0 | Train Loss: 7.2660 | Train Loss Unweighted: 2.6454 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3588 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1251 | Regression_eta: 0.0097 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6358 | Classification_binary: 2.3588 | Regression_cos_phi: 0.4704 | Regression_energy: 1.3680 | Regression_eta: 0.4329 | Regression_pt: 1.4467 | Regression_sin_phi: 0.5534 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3536s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0169 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.2688 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6467.5 | uss_MB: 5997.8 | child_rss_MB: 3185.5 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:47,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:58,063] INFO : Step 49400/50000 rank0 | Train Loss: 8.1874 | Train Loss Unweighted: 2.9322 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6051 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1438 | Regression_eta: 0.0119 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7466 | Classification_binary: 2.6051 | Regression_cos_phi: 0.4762 | Regression_energy: 1.5725 | Regression_eta: 0.5301 | Regression_pt: 1.6360 | Regression_sin_phi: 0.6210 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1039 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.3042 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6467.5 | uss_MB: 5997.8 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:58,276] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:09,176] INFO : Step 49500/50000 rank0 | Train Loss: 7.4606 | Train Loss Unweighted: 2.8836 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5962 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1252 | Regression_eta: 0.0089 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.5962 | Regression_cos_phi: 0.4405 | Regression_energy: 1.3686 | Regression_eta: 0.3975 | Regression_pt: 1.4642 | Regression_sin_phi: 0.5237 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4896s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0867 | max_loss: 0.0035 | max_backward: 0.0388 | max_optimizer_step: 0.1441 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6467.5 | uss_MB: 5999.9 | child_rss_MB: 3187.3 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:25:09,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:20,242] INFO : Step 49600/50000 rank0 | Train Loss: 8.0624 | Train Loss Unweighted: 2.8580 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5260 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1493 | Regression_eta: 0.0102 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7390 | Classification_binary: 2.5260 | Regression_cos_phi: 0.4297 | Regression_energy: 1.6317 | Regression_eta: 0.4530 | Regression_pt: 1.6606 | Regression_sin_phi: 0.6224 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4769s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1004 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3120 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6349.7 | uss_MB: 5879.9 | child_rss_MB: 3187.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:25:20,456] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:31,434] INFO : Step 49700/50000 rank0 | Train Loss: 7.6107 | Train Loss Unweighted: 2.8008 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5044 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1294 | Regression_eta: 0.0099 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6974 | Classification_binary: 2.5044 | Regression_cos_phi: 0.3312 | Regression_energy: 1.4147 | Regression_eta: 0.4419 | Regression_pt: 1.4943 | Regression_sin_phi: 0.7268 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4641s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1051 | max_forward: 0.0141 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1438 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6349.7 | uss_MB: 5880.4 | child_rss_MB: 3187.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:25:31,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:42,442] INFO : Step 49800/50000 rank0 | Train Loss: 8.2056 | Train Loss Unweighted: 2.8163 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4889 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1438 | Regression_eta: 0.0225 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6846 | Classification_binary: 2.4889 | Regression_cos_phi: 0.4349 | Regression_energy: 1.5725 | Regression_eta: 1.0052 | Regression_pt: 1.5591 | Regression_sin_phi: 0.4604 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4444s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0560 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1428 | max_tensorboard_logging: 0.3104 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6349.7 | uss_MB: 5880.5 | child_rss_MB: 3187.0 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:25:42,654] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:53,593] INFO : Step 49900/50000 rank0 | Train Loss: 7.9661 | Train Loss Unweighted: 2.8305 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4954 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1474 | Regression_eta: 0.0125 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6782 | Classification_binary: 2.4954 | Regression_cos_phi: 0.3538 | Regression_energy: 1.6117 | Regression_eta: 0.5585 | Regression_pt: 1.7350 | Regression_sin_phi: 0.5335 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0438 | max_forward: 0.0161 | max_loss: 0.0037 | max_backward: 0.0380 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.2548 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6349.7 | uss_MB: 5882.7 | child_rss_MB: 3188.9 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:25:53,806] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:04,718] INFO : Step 50000/50000 rank0 | Train Loss: 7.9695 | Train Loss Unweighted: 2.8070 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4928 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1375 | Regression_eta: 0.0136 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6647 | Classification_binary: 2.4928 | Regression_cos_phi: 0.4480 | Regression_energy: 1.5033 | Regression_eta: 0.6055 | Regression_pt: 1.5595 | Regression_sin_phi: 0.6957 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3528s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0035 | max_backward: 0.0342 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.2693 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6349.7 | uss_MB: 5880.7 | child_rss_MB: 3190.6 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:26:04,936] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:04,954] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 13:26:05,039] INFO : Running validation on rank0 +rank0 [2026-10-28 13:26:05,039] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:26:05,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:26:06,141] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 13:26:06,288] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:26:06,422] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:26:06,497] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:26:06,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:26:06,634] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:26:06,704] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:26:06,775] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:26:06,886] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:26:07,019] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:26:07,152] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:26:07,223] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:26:07,294] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:26:07,362] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:26:07,432] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:26:07,502] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:26:07,619] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:26:07,700] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:26:07,703] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:26:07,707] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:26:07,709] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:26:07,712] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:26:07,713] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:26:07,715] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:26:07,718] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:26:07,720] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:26:07,723] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:26:07,726] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:26:07,728] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:26:07,731] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:26:07,734] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:26:07,735] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:26:13,675] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.9695 | Valid Loss=7.8414 | Train Loss Unweighted=2.8070 | Valid Loss Unweighted=2.8376 | Stale=2 | ETA=0.0m +rank0 [2026-10-28 13:26:13,685] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:13,743] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:26:13,747] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:13,935] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:15,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:16,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:16,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:17,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:17,596] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:26:17,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:17,654] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:26:17,657] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:17,786] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:19,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:20,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:20,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:21,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:21,344] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:26:21,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:21,401] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:26:21,404] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:26:21,541] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:23,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:23,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:24,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:24,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:24,876] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:26:24,883] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:25,147] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:25,561] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:27,004] INFO : Plotted jets +rank0 [2026-10-28 13:26:28,095] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:28,127] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:28,362] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:28,755] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:30,143] INFO : Plotted jets +rank0 [2026-10-28 13:26:31,807] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:31,840] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:32,051] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:32,415] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:33,788] INFO : Plotted jets +rank0 [2026-10-28 13:26:34,850] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:35,316] INFO : Training completed. Total time on device 0: 134.032min diff --git a/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train.log.0 b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..3e060e14db4d5c78502ccc7a506074f35b4ee5c3 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_attention_seed12345_20261028_111201_967206/train.log.0 @@ -0,0 +1,13847 @@ +rank0 [2026-10-28 11:12:24,877] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,877] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,878] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,878] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,878] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,915] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,915] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,915] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,915] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,915] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,918] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,918] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,921] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,921] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,924] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,924] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,927] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:24,927] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,929] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:24,929] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,932] INFO : Backbone initialization took 0.02s +rank0 [2026-10-28 11:12:24,932] INFO : backbone parameters: 2374656 +rank0 [2026-10-28 11:12:24,933] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:24,937] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:24,937] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:24,937] INFO : jagged_attention enabled=True +rank0 [2026-10-28 11:12:24,938] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-28 11:12:24,938] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:24,938] INFO : classification_norm parameters: 512 +rank0 [2026-10-28 11:12:24,938] INFO : regression_norm parameters: 512 +rank0 [2026-10-28 11:12:24,938] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:24,938] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:24,938] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:24,938] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:24,939] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:24,939] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:24,939] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,939] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,939] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:24,939] INFO : Total MLPF parameters: 3114258 +rank0 [2026-10-28 11:12:24,939] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:24,940] INFO : Instantiated model +rank0 [2026-10-28 11:12:24,940] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,313] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,314] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,394] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +131 nn_energy.nn.1.0.bias 256 0 +132 nn_energy.nn.1.2.weight 256 0 +133 nn_energy.nn.1.2.bias 256 0 +134 nn_energy.nn.1.4.weight 256 0 +135 nn_energy.nn.1.4.bias 1 0 + +[136 rows x 3 columns] +rank0 [2026-10-28 11:12:25,394] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,396] INFO : Trainable parameters: 3114258 +rank0 [2026-10-28 11:12:25,396] INFO : Non-trainable parameters: 0 +rank0 [2026-10-28 11:12:25,396] INFO : Total parameters: 3114258 +rank0 [2026-10-28 11:12:25,398] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 +backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 +backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 +backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 +backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 +backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 +backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,401] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206 +rank0 [2026-10-28 11:12:25,401] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206 +rank0 [2026-10-28 11:12:25,401] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,401] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,271] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,312] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,343] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,376] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,402] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,441] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,464] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,490] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,515] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,541] INFO : train_dataset: cld_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,541] INFO : sample=cld_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,561] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,596] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,647] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,694] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,733] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,762] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,789] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,815] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,839] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,865] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,865] INFO : sample=cld_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,898] INFO : train_dataset: cld_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:32,922] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,966] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,994] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,017] INFO : train_dataset: cld_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,053] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,080] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,122] INFO : train_dataset: cld_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,148] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset: cld_edm_qq_pf, 90598 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,177] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,177] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,177] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,183] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,189] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,195] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,200] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,205] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,211] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,216] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,222] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,227] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,233] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,233] INFO : sample=cld_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,238] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,243] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,248] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,253] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,258] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,264] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,269] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,274] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,279] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,284] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,284] INFO : sample=cld_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,289] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,294] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,300] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,305] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,310] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,315] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,320] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,325] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,330] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,335] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,336] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,336] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,370] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,373] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,374] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-28 11:12:33,374] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,374] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,380] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,380] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,397] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,640] INFO : Created train_iterator +rank0 [2026-10-28 11:13:37,530] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.284732975959778, 'Classification': 0.140388917773962, 'Regression_pt': 0.4447642934322357, 'Regression_eta': 0.11849542165175081, 'Regression_sin_phi': 0.055837437864393, 'Regression_cos_phi': 0.08143645193427801, 'Regression_energy': 0.4833901843428612}, weights={'Classification_binary': 1.0, 'Classification': 37.64352035522461, 'Regression_pt': 11.882098197937012, 'Regression_eta': 44.59862518310547, 'Regression_sin_phi': 94.64497375488281, 'Regression_cos_phi': 64.89395141601562, 'Regression_energy': 10.93264389038086} +rank0 [2026-10-28 11:13:39,439] INFO : Step 100/50000 rank0 | Train Loss: 5.7439 | Train Loss Unweighted: 5.7439 | Train Loss Components: Classification: 0.0486 | Classification_binary: 4.8688 | Regression_cos_phi: 0.0220 | Regression_energy: 0.3793 | Regression_eta: 0.0353 | Regression_pt: 0.3669 | Regression_sin_phi: 0.0231 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 4.8688 | Regression_cos_phi: 0.0220 | Regression_energy: 0.3793 | Regression_eta: 0.0353 | Regression_pt: 0.3669 | Regression_sin_phi: 0.0231 | LR: 1.00e-03 | DataLoad Time: 0.0553s | Model Forward Time: 2.2113s | Phase Times: max_pre_step_barrier: 0.0005 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.1093 | max_backward: 0.0335 | max_optimizer_step: 0.1019 | max_tensorboard_logging: 1.2837 | max_tensorboard_flush: 0.0698 | Memory: rss_MB: 2689.1 | uss_MB: 1984.0 | child_rss_MB: 3276.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:13:39,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 97.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:02,256] INFO : Step 200/50000 rank0 | Train Loss: 18.4973 | Train Loss Unweighted: 5.5715 | Train Loss Components: Classification: 0.0446 | Classification_binary: 4.7553 | Regression_cos_phi: 0.0150 | Regression_energy: 0.3632 | Regression_eta: 0.0322 | Regression_pt: 0.3443 | Regression_sin_phi: 0.0168 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.6797 | Classification_binary: 4.7553 | Regression_cos_phi: 0.9765 | Regression_energy: 3.9711 | Regression_eta: 1.4367 | Regression_pt: 4.0912 | Regression_sin_phi: 1.5868 | LR: 1.00e-03 | DataLoad Time: 0.0015s | Model Forward Time: 0.9800s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0139 | max_loss: 0.0030 | max_backward: 0.0331 | max_optimizer_step: 0.0991 | max_tensorboard_logging: 0.2484 | max_tensorboard_flush: 0.0083 | Memory: rss_MB: 2688.7 | uss_MB: 1985.5 | child_rss_MB: 3280.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:14:02,468] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 105.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:23,115] INFO : Step 300/50000 rank0 | Train Loss: 18.2072 | Train Loss Unweighted: 5.6801 | Train Loss Components: Classification: 0.0399 | Classification_binary: 4.8508 | Regression_cos_phi: 0.0148 | Regression_energy: 0.3696 | Regression_eta: 0.0277 | Regression_pt: 0.3635 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5038 | Classification_binary: 4.8508 | Regression_cos_phi: 0.9608 | Regression_energy: 4.0408 | Regression_eta: 1.2365 | Regression_pt: 4.3193 | Regression_sin_phi: 1.2954 | LR: 1.00e-03 | DataLoad Time: 0.0019s | Model Forward Time: 1.1496s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0833 | max_forward: 0.0132 | max_loss: 0.0033 | max_backward: 0.0317 | max_optimizer_step: 0.0966 | max_tensorboard_logging: 0.3505 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 2688.4 | uss_MB: 1986.2 | child_rss_MB: 3276.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:14:23,325] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 105.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:43,387] INFO : Step 400/50000 rank0 | Train Loss: 16.8231 | Train Loss Unweighted: 5.3455 | Train Loss Components: Classification: 0.0407 | Classification_binary: 4.5716 | Regression_cos_phi: 0.0122 | Regression_energy: 0.3465 | Regression_eta: 0.0180 | Regression_pt: 0.3432 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5309 | Classification_binary: 4.5716 | Regression_cos_phi: 0.7946 | Regression_energy: 3.7878 | Regression_eta: 0.8007 | Regression_pt: 4.0779 | Regression_sin_phi: 1.2596 | LR: 1.00e-03 | DataLoad Time: 0.0514s | Model Forward Time: 0.9896s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 2688.1 | uss_MB: 1986.7 | child_rss_MB: 3276.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:14:43,599] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:03,132] INFO : Step 500/50000 rank0 | Train Loss: 17.1555 | Train Loss Unweighted: 5.3923 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.6286 | Regression_cos_phi: 0.0142 | Regression_energy: 0.3379 | Regression_eta: 0.0253 | Regression_pt: 0.3321 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5174 | Classification_binary: 4.6286 | Regression_cos_phi: 0.9193 | Regression_energy: 3.6937 | Regression_eta: 1.1282 | Regression_pt: 3.9459 | Regression_sin_phi: 1.3223 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.0920s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0159 | max_loss: 0.0030 | max_backward: 0.0381 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.2593 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 2690.0 | uss_MB: 1989.0 | child_rss_MB: 3277.4 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:15:03,350] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:22,123] INFO : Step 600/50000 rank0 | Train Loss: 17.4910 | Train Loss Unweighted: 5.3451 | Train Loss Components: Classification: 0.0374 | Classification_binary: 4.5458 | Regression_cos_phi: 0.0137 | Regression_energy: 0.3606 | Regression_eta: 0.0381 | Regression_pt: 0.3392 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4074 | Classification_binary: 4.5458 | Regression_cos_phi: 0.8911 | Regression_energy: 3.9423 | Regression_eta: 1.6994 | Regression_pt: 4.0307 | Regression_sin_phi: 0.9744 | LR: 1.00e-03 | DataLoad Time: 0.0016s | Model Forward Time: 1.0824s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0163 | max_loss: 0.0030 | max_backward: 0.1418 | max_optimizer_step: 0.0988 | max_tensorboard_logging: 0.2480 | max_tensorboard_flush: 0.0079 | Memory: rss_MB: 2623.3 | uss_MB: 1921.6 | child_rss_MB: 3274.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:15:22,352] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:41,530] INFO : Step 700/50000 rank0 | Train Loss: 16.5769 | Train Loss Unweighted: 5.3617 | Train Loss Components: Classification: 0.0386 | Classification_binary: 4.6112 | Regression_cos_phi: 0.0115 | Regression_energy: 0.3339 | Regression_eta: 0.0194 | Regression_pt: 0.3335 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4522 | Classification_binary: 4.6112 | Regression_cos_phi: 0.7476 | Regression_energy: 3.6505 | Regression_eta: 0.8641 | Regression_pt: 3.9629 | Regression_sin_phi: 1.2884 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.0783s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0029 | max_backward: 0.0380 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.2494 | max_tensorboard_flush: 0.0079 | Memory: rss_MB: 2627.4 | uss_MB: 1927.1 | child_rss_MB: 3266.4 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:15:41,746] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:00,427] INFO : Step 800/50000 rank0 | Train Loss: 16.1142 | Train Loss Unweighted: 5.2348 | Train Loss Components: Classification: 0.0371 | Classification_binary: 4.5277 | Regression_cos_phi: 0.0125 | Regression_energy: 0.3125 | Regression_eta: 0.0205 | Regression_pt: 0.3100 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3981 | Classification_binary: 4.5277 | Regression_cos_phi: 0.8137 | Regression_energy: 3.4167 | Regression_eta: 0.9155 | Regression_pt: 3.6837 | Regression_sin_phi: 1.3589 | LR: 9.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0135 | max_loss: 0.0029 | max_backward: 0.0330 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.2715 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 2626.2 | uss_MB: 1926.3 | child_rss_MB: 3272.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:16:00,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:19,403] INFO : Step 900/50000 rank0 | Train Loss: 15.1225 | Train Loss Unweighted: 5.2828 | Train Loss Components: Classification: 0.0363 | Classification_binary: 4.6219 | Regression_cos_phi: 0.0111 | Regression_energy: 0.2949 | Regression_eta: 0.0171 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3654 | Classification_binary: 4.6219 | Regression_cos_phi: 0.7179 | Regression_energy: 3.2241 | Regression_eta: 0.7610 | Regression_pt: 3.4620 | Regression_sin_phi: 0.9702 | LR: 9.99e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1217s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0158 | max_loss: 0.0031 | max_backward: 0.0346 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.2938 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 2628.9 | uss_MB: 1928.9 | child_rss_MB: 3267.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:16:19,626] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:38,580] INFO : Step 1000/50000 rank0 | Train Loss: 16.1139 | Train Loss Unweighted: 5.2604 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.5681 | Regression_cos_phi: 0.0140 | Regression_energy: 0.3061 | Regression_eta: 0.0219 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.5186 | Classification_binary: 4.5681 | Regression_cos_phi: 0.9086 | Regression_energy: 3.3466 | Regression_eta: 0.9780 | Regression_pt: 3.5238 | Regression_sin_phi: 1.2702 | LR: 9.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0038s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0131 | max_loss: 0.0029 | max_backward: 0.0330 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 2629.0 | uss_MB: 1929.7 | child_rss_MB: 3266.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:16:38,794] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:38,902] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:16:38,990] INFO : Running validation on rank0 +rank0 [2026-10-28 11:16:38,990] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:16:38,990] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:16:40,188] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:16:40,374] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:16:40,471] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:16:40,532] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:16:40,591] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:16:40,642] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:16:40,701] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:16:40,763] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:16:40,820] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:16:40,907] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:16:40,995] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:16:41,054] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:16:41,114] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:16:41,164] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:16:41,224] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:16:41,285] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:16:41,343] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:16:41,347] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:16:41,350] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:16:41,353] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:16:41,353] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:16:41,356] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:16:41,359] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:16:41,359] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:16:41,362] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:16:41,365] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:16:41,365] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:16:41,368] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:16:41,370] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:16:41,371] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:16:41,374] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:16:41,377] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:16:47,276] INFO : VALIDATION | Step=1000/50000 | Train Loss=16.1139 | Valid Loss=15.5723 | Train Loss Unweighted=5.2604 | Valid Loss Unweighted=5.2125 | Stale=0 | ETA=207.3m +rank0 [2026-10-28 11:16:47,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:47,346] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:16:47,356] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:16:47,489] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:49,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:49,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:50,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:51,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:51,239] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:16:51,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:51,297] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:16:51,300] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:16:51,367] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:52,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:53,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:53,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:54,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:54,596] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:54,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:54,651] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:16:54,654] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:16:54,722] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:56,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:56,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:57,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:57,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:57,712] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:57,721] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:16:57,978] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:16:58,882] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:00,379] INFO : Plotted jets +rank0 [2026-10-28 11:17:01,461] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:01,494] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:01,721] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:02,111] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:03,497] INFO : Plotted jets +rank0 [2026-10-28 11:17:04,598] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:04,634] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:17:05,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:17:05,489] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:06,850] INFO : Plotted jets +rank0 [2026-10-28 11:17:07,886] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:26,618] INFO : Step 1100/50000 rank0 | Train Loss: 15.8770 | Train Loss Unweighted: 5.1017 | Train Loss Components: Classification: 0.0383 | Classification_binary: 4.4144 | Regression_cos_phi: 0.0126 | Regression_energy: 0.3013 | Regression_eta: 0.0295 | Regression_pt: 0.2941 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4406 | Classification_binary: 4.4144 | Regression_cos_phi: 0.8204 | Regression_energy: 3.2940 | Regression_eta: 1.3138 | Regression_pt: 3.4940 | Regression_sin_phi: 1.0997 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.0090s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0162 | max_loss: 0.0030 | max_backward: 0.0340 | max_optimizer_step: 0.0976 | max_tensorboard_logging: 0.2568 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 2894.5 | uss_MB: 2380.7 | child_rss_MB: 3264.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:17:26,832] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:45,456] INFO : Step 1200/50000 rank0 | Train Loss: 15.2241 | Train Loss Unweighted: 5.0294 | Train Loss Components: Classification: 0.0381 | Classification_binary: 4.3872 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2798 | Regression_eta: 0.0218 | Regression_pt: 0.2764 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4332 | Classification_binary: 4.3872 | Regression_cos_phi: 0.8240 | Regression_energy: 3.0593 | Regression_eta: 0.9709 | Regression_pt: 3.2842 | Regression_sin_phi: 1.2653 | LR: 9.99e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.0210s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0145 | max_loss: 0.0031 | max_backward: 0.0334 | max_optimizer_step: 0.1016 | max_tensorboard_logging: 0.2635 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2895.4 | uss_MB: 2381.5 | child_rss_MB: 3260.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:17:45,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:03,944] INFO : Step 1300/50000 rank0 | Train Loss: 16.5995 | Train Loss Unweighted: 5.1553 | Train Loss Components: Classification: 0.0379 | Classification_binary: 4.4367 | Regression_cos_phi: 0.0140 | Regression_energy: 0.3180 | Regression_eta: 0.0358 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4248 | Classification_binary: 4.4367 | Regression_cos_phi: 0.9105 | Regression_energy: 3.4767 | Regression_eta: 1.5985 | Regression_pt: 3.5686 | Regression_sin_phi: 1.1836 | LR: 9.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.0529s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0173 | max_loss: 0.0458 | max_backward: 0.0381 | max_optimizer_step: 0.0987 | max_tensorboard_logging: 0.2586 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2897.8 | uss_MB: 2384.4 | child_rss_MB: 3259.3 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:18:04,158] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:22,241] INFO : Step 1400/50000 rank0 | Train Loss: 15.6765 | Train Loss Unweighted: 5.0366 | Train Loss Components: Classification: 0.0354 | Classification_binary: 4.3577 | Regression_cos_phi: 0.0103 | Regression_energy: 0.3028 | Regression_eta: 0.0285 | Regression_pt: 0.2879 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3338 | Classification_binary: 4.3577 | Regression_cos_phi: 0.6711 | Regression_energy: 3.3106 | Regression_eta: 1.2729 | Regression_pt: 3.4213 | Regression_sin_phi: 1.3093 | LR: 9.98e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.1322s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0541 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1000 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 2895.2 | uss_MB: 2381.7 | child_rss_MB: 3263.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:22,452] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:40,629] INFO : Step 1500/50000 rank0 | Train Loss: 14.8835 | Train Loss Unweighted: 5.0770 | Train Loss Components: Classification: 0.0360 | Classification_binary: 4.4380 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2851 | Regression_eta: 0.0205 | Regression_pt: 0.2749 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3537 | Classification_binary: 4.4380 | Regression_cos_phi: 0.7368 | Regression_energy: 3.1164 | Regression_eta: 0.9129 | Regression_pt: 3.2668 | Regression_sin_phi: 1.0588 | LR: 9.98e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.0852s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0031 | max_backward: 0.0327 | max_optimizer_step: 0.1007 | max_tensorboard_logging: 0.3089 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 2896.2 | uss_MB: 2383.5 | child_rss_MB: 3262.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:18:40,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:59,087] INFO : Step 1600/50000 rank0 | Train Loss: 15.6863 | Train Loss Unweighted: 5.0281 | Train Loss Components: Classification: 0.0352 | Classification_binary: 4.3291 | Regression_cos_phi: 0.0130 | Regression_energy: 0.3119 | Regression_eta: 0.0229 | Regression_pt: 0.3037 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3262 | Classification_binary: 4.3291 | Regression_cos_phi: 0.8432 | Regression_energy: 3.4103 | Regression_eta: 1.0220 | Regression_pt: 3.6090 | Regression_sin_phi: 1.1465 | LR: 9.98e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1070s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0030 | max_backward: 0.1295 | max_optimizer_step: 0.1032 | max_tensorboard_logging: 0.2557 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2792.1 | uss_MB: 2278.7 | child_rss_MB: 3258.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:59,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:17,632] INFO : Step 1700/50000 rank0 | Train Loss: 14.0979 | Train Loss Unweighted: 4.9267 | Train Loss Components: Classification: 0.0373 | Classification_binary: 4.3204 | Regression_cos_phi: 0.0094 | Regression_energy: 0.2711 | Regression_eta: 0.0208 | Regression_pt: 0.2594 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4023 | Classification_binary: 4.3204 | Regression_cos_phi: 0.6120 | Regression_energy: 2.9634 | Regression_eta: 0.9255 | Regression_pt: 3.0828 | Regression_sin_phi: 0.7915 | LR: 9.97e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.9851s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0030 | max_backward: 0.0331 | max_optimizer_step: 0.1006 | max_tensorboard_logging: 0.2450 | max_tensorboard_flush: 0.0080 | Memory: rss_MB: 2794.1 | uss_MB: 2281.4 | child_rss_MB: 3263.9 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:19:17,845] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:36,529] INFO : Step 1800/50000 rank0 | Train Loss: 15.1663 | Train Loss Unweighted: 5.0728 | Train Loss Components: Classification: 0.0393 | Classification_binary: 4.4279 | Regression_cos_phi: 0.0121 | Regression_energy: 0.2843 | Regression_eta: 0.0230 | Regression_pt: 0.2748 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4806 | Classification_binary: 4.4279 | Regression_cos_phi: 0.7833 | Regression_energy: 3.1083 | Regression_eta: 1.0265 | Regression_pt: 3.2651 | Regression_sin_phi: 1.0746 | LR: 9.97e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1981s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0597 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1064 | max_tensorboard_logging: 0.3634 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2794.1 | uss_MB: 2281.4 | child_rss_MB: 3257.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:19:36,740] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:55,114] INFO : Step 1900/50000 rank0 | Train Loss: 14.0118 | Train Loss Unweighted: 4.9192 | Train Loss Components: Classification: 0.0345 | Classification_binary: 4.3004 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2760 | Regression_eta: 0.0131 | Regression_pt: 0.2762 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2986 | Classification_binary: 4.3004 | Regression_cos_phi: 0.5986 | Regression_energy: 3.0173 | Regression_eta: 0.5823 | Regression_pt: 3.2816 | Regression_sin_phi: 0.9329 | LR: 9.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.0819s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0166 | max_loss: 0.0028 | max_backward: 0.0383 | max_optimizer_step: 0.1905 | max_tensorboard_logging: 0.2514 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2797.7 | uss_MB: 2286.1 | child_rss_MB: 3253.2 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:19:55,328] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:13,387] INFO : Step 2000/50000 rank0 | Train Loss: 15.0794 | Train Loss Unweighted: 5.0277 | Train Loss Components: Classification: 0.0378 | Classification_binary: 4.3431 | Regression_cos_phi: 0.0113 | Regression_energy: 0.3091 | Regression_eta: 0.0165 | Regression_pt: 0.3004 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4222 | Classification_binary: 4.3431 | Regression_cos_phi: 0.7329 | Regression_energy: 3.3791 | Regression_eta: 0.7370 | Regression_pt: 3.5700 | Regression_sin_phi: 0.8951 | LR: 9.96e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.0935s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0136 | max_loss: 0.0031 | max_backward: 0.1263 | max_optimizer_step: 0.1027 | max_tensorboard_logging: 0.2513 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2796.1 | uss_MB: 2283.9 | child_rss_MB: 3256.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:20:13,614] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:13,631] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:20:13,713] INFO : Running validation on rank0 +rank0 [2026-10-28 11:20:13,713] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:20:13,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:20:14,899] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:20:15,041] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:20:15,172] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:20:15,244] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:20:15,312] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:20:15,378] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:20:15,446] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:20:15,515] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:20:15,615] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:20:15,735] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:20:15,856] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:20:15,927] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:20:15,995] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:20:16,062] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:20:16,130] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:20:16,199] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:20:16,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:20:16,312] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:20:16,315] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:20:16,318] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:20:16,319] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:20:16,321] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:20:16,324] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:20:16,325] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:20:16,327] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:20:16,330] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:20:16,333] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:20:16,335] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:20:16,337] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:20:16,339] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:20:16,342] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:20:16,344] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:20:22,186] INFO : VALIDATION | Step=2000/50000 | Train Loss=15.0794 | Valid Loss=14.4329 | Train Loss Unweighted=5.0277 | Valid Loss Unweighted=4.9395 | Stale=0 | ETA=187.5m +rank0 [2026-10-28 11:20:22,195] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:22,255] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:20:22,259] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:22,403] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:23,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:24,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:25,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:25,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:25,816] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:25,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:25,873] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:25,876] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:25,941] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:27,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:27,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:28,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:29,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:29,152] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:29,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:29,210] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:29,213] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:20:29,286] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:30,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:31,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:31,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:32,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:32,270] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:32,277] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:32,539] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:32,941] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:34,380] INFO : Plotted jets +rank0 [2026-10-28 11:20:35,460] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:35,493] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:35,722] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:36,496] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:37,912] INFO : Plotted jets +rank0 [2026-10-28 11:20:39,003] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:39,036] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:39,239] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:39,597] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:40,965] INFO : Plotted jets +rank0 [2026-10-28 11:20:42,009] INFO : Plotted jet ratio +rank0 [2026-10-28 11:21:00,513] INFO : Step 2100/50000 rank0 | Train Loss: 15.2360 | Train Loss Unweighted: 5.0142 | Train Loss Components: Classification: 0.0377 | Classification_binary: 4.3463 | Regression_cos_phi: 0.0125 | Regression_energy: 0.3000 | Regression_eta: 0.0163 | Regression_pt: 0.2884 | Regression_sin_phi: 0.0130 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4184 | Classification_binary: 4.3463 | Regression_cos_phi: 0.8085 | Regression_energy: 3.2801 | Regression_eta: 0.7280 | Regression_pt: 3.4265 | Regression_sin_phi: 1.2281 | LR: 9.96e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.0428s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.1015 | max_tensorboard_logging: 0.2759 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3018.4 | uss_MB: 2505.5 | child_rss_MB: 3244.0 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:21:00,727] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:18,664] INFO : Step 2200/50000 rank0 | Train Loss: 13.5599 | Train Loss Unweighted: 4.8168 | Train Loss Components: Classification: 0.0383 | Classification_binary: 4.2581 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2467 | Regression_eta: 0.0173 | Regression_pt: 0.2362 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.4428 | Classification_binary: 4.2581 | Regression_cos_phi: 0.7080 | Regression_energy: 2.6966 | Regression_eta: 0.7699 | Regression_pt: 2.8071 | Regression_sin_phi: 0.8774 | LR: 9.96e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.0685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0160 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1039 | max_tensorboard_logging: 0.2707 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3018.9 | uss_MB: 2508.1 | child_rss_MB: 3241.4 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:21:18,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:36,726] INFO : Step 2300/50000 rank0 | Train Loss: 13.7804 | Train Loss Unweighted: 4.8773 | Train Loss Components: Classification: 0.0335 | Classification_binary: 4.2668 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2773 | Regression_eta: 0.0172 | Regression_pt: 0.2662 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2629 | Classification_binary: 4.2668 | Regression_cos_phi: 0.5526 | Regression_energy: 3.0311 | Regression_eta: 0.7682 | Regression_pt: 3.1631 | Regression_sin_phi: 0.7357 | LR: 9.95e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1982s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0879 | max_forward: 0.0150 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.0999 | max_tensorboard_logging: 0.3581 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 3019.4 | uss_MB: 2506.5 | child_rss_MB: 3238.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:21:36,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:54,814] INFO : Step 2400/50000 rank0 | Train Loss: 14.2686 | Train Loss Unweighted: 4.8262 | Train Loss Components: Classification: 0.0322 | Classification_binary: 4.2091 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2748 | Regression_eta: 0.0159 | Regression_pt: 0.2703 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2106 | Classification_binary: 4.2091 | Regression_cos_phi: 0.7286 | Regression_energy: 3.0047 | Regression_eta: 0.7069 | Regression_pt: 3.2119 | Regression_sin_phi: 1.1968 | LR: 9.95e-04 | DataLoad Time: 0.0820s | Model Forward Time: 1.1696s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0742 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1003 | max_tensorboard_logging: 0.3536 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 3019.4 | uss_MB: 2505.3 | child_rss_MB: 3239.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:21:55,027] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:13,024] INFO : Step 2500/50000 rank0 | Train Loss: 13.4616 | Train Loss Unweighted: 4.7062 | Train Loss Components: Classification: 0.0347 | Classification_binary: 4.1132 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2645 | Regression_eta: 0.0120 | Regression_pt: 0.2632 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.3070 | Classification_binary: 4.1132 | Regression_cos_phi: 0.6033 | Regression_energy: 2.8913 | Regression_eta: 0.5356 | Regression_pt: 3.1271 | Regression_sin_phi: 0.8840 | LR: 9.94e-04 | DataLoad Time: 0.1600s | Model Forward Time: 1.1785s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0544 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0340 | max_optimizer_step: 0.1007 | max_tensorboard_logging: 0.3547 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 3020.4 | uss_MB: 2507.2 | child_rss_MB: 3240.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:22:13,235] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:31,263] INFO : Step 2600/50000 rank0 | Train Loss: 12.8478 | Train Loss Unweighted: 4.6747 | Train Loss Components: Classification: 0.0327 | Classification_binary: 4.1557 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2277 | Regression_eta: 0.0131 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2321 | Classification_binary: 4.1557 | Regression_cos_phi: 0.5823 | Regression_energy: 2.4894 | Regression_eta: 0.5857 | Regression_pt: 2.6670 | Regression_sin_phi: 1.1356 | LR: 9.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2777s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0887 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0385 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.3603 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 3020.4 | uss_MB: 2509.5 | child_rss_MB: 3234.5 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:22:31,477] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:49,360] INFO : Step 2700/50000 rank0 | Train Loss: 14.1400 | Train Loss Unweighted: 4.8479 | Train Loss Components: Classification: 0.0331 | Classification_binary: 4.2176 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2833 | Regression_eta: 0.0135 | Regression_pt: 0.2793 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2448 | Classification_binary: 4.2176 | Regression_cos_phi: 0.7333 | Regression_energy: 3.0968 | Regression_eta: 0.6017 | Regression_pt: 3.3188 | Regression_sin_phi: 0.9269 | LR: 9.94e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2529s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0534 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.4032 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 2884.9 | uss_MB: 2373.5 | child_rss_MB: 3241.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:22:49,575] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:07,671] INFO : Step 2800/50000 rank0 | Train Loss: 13.1247 | Train Loss Unweighted: 4.7647 | Train Loss Components: Classification: 0.0322 | Classification_binary: 4.2320 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2347 | Regression_eta: 0.0182 | Regression_pt: 0.2277 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2111 | Classification_binary: 4.2320 | Regression_cos_phi: 0.6339 | Regression_energy: 2.5664 | Regression_eta: 0.8130 | Regression_pt: 2.7051 | Regression_sin_phi: 0.9632 | LR: 9.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1158s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0031 | max_backward: 0.0337 | max_optimizer_step: 0.1016 | max_tensorboard_logging: 0.3043 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 2885.9 | uss_MB: 2374.0 | child_rss_MB: 3239.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:23:07,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:25,584] INFO : Step 2900/50000 rank0 | Train Loss: 12.5833 | Train Loss Unweighted: 4.7099 | Train Loss Components: Classification: 0.0311 | Classification_binary: 4.1574 | Regression_cos_phi: 0.0070 | Regression_energy: 0.2491 | Regression_eta: 0.0104 | Regression_pt: 0.2478 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1708 | Classification_binary: 4.1574 | Regression_cos_phi: 0.4525 | Regression_energy: 2.7235 | Regression_eta: 0.4654 | Regression_pt: 2.9450 | Regression_sin_phi: 0.6687 | LR: 9.93e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2346s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0728 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.0995 | max_tensorboard_logging: 0.3855 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 2886.9 | uss_MB: 2375.2 | child_rss_MB: 3239.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:23:25,799] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:43,434] INFO : Step 3000/50000 rank0 | Train Loss: 13.6555 | Train Loss Unweighted: 4.7133 | Train Loss Components: Classification: 0.0325 | Classification_binary: 4.1296 | Regression_cos_phi: 0.0128 | Regression_energy: 0.2575 | Regression_eta: 0.0139 | Regression_pt: 0.2565 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2238 | Classification_binary: 4.1296 | Regression_cos_phi: 0.8335 | Regression_energy: 2.8155 | Regression_eta: 0.6212 | Regression_pt: 3.0475 | Regression_sin_phi: 0.9844 | LR: 9.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1645s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0028 | max_backward: 0.1233 | max_optimizer_step: 0.1014 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 2887.9 | uss_MB: 2378.0 | child_rss_MB: 3240.3 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 11:23:43,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:43,665] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:23:43,746] INFO : Running validation on rank0 +rank0 [2026-10-28 11:23:43,746] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:23:43,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:23:44,935] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:23:45,078] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:23:45,209] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:23:45,281] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:23:45,349] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:23:45,417] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:23:45,486] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:23:45,556] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:23:45,655] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:23:45,776] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:23:45,897] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:23:45,967] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:23:46,035] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:23:46,102] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:23:46,172] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:23:46,241] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:23:46,350] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:23:46,357] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:23:46,360] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:23:46,361] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:23:46,364] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:23:46,364] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:23:46,367] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:23:46,370] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:23:46,371] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:23:46,373] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:23:46,374] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:23:46,377] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:23:46,380] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:23:46,380] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:23:46,383] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:23:46,384] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:23:52,338] INFO : VALIDATION | Step=3000/50000 | Train Loss=13.6555 | Valid Loss=13.5005 | Train Loss Unweighted=4.7133 | Valid Loss Unweighted=4.7697 | Stale=0 | ETA=177.3m +rank0 [2026-10-28 11:23:52,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:52,404] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:23:52,407] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:23:52,450] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:53,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:54,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:55,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:55,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:55,804] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:23:55,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:55,862] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:23:55,865] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:55,933] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:57,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:57,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:58,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:59,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:59,113] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:23:59,115] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:59,174] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:23:59,177] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:23:59,249] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:24:00,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:24:01,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:24:01,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:24:02,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:24:02,352] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:24:02,360] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:24:02,613] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:24:03,020] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:04,458] INFO : Plotted jets +rank0 [2026-10-28 11:24:05,543] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:05,575] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:24:05,801] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:24:06,574] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:07,960] INFO : Plotted jets +rank0 [2026-10-28 11:24:09,063] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:09,096] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:24:09,297] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:24:09,658] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:11,031] INFO : Plotted jets +rank0 [2026-10-28 11:24:12,081] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:30,059] INFO : Step 3100/50000 rank0 | Train Loss: 13.7812 | Train Loss Unweighted: 4.7819 | Train Loss Components: Classification: 0.0338 | Classification_binary: 4.2095 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2541 | Regression_eta: 0.0219 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2730 | Classification_binary: 4.2095 | Regression_cos_phi: 0.6530 | Regression_energy: 2.7781 | Regression_eta: 0.9765 | Regression_pt: 2.8723 | Regression_sin_phi: 1.0188 | LR: 9.91e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1537 | max_tensorboard_logging: 0.2888 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 3107.0 | uss_MB: 2595.8 | child_rss_MB: 3237.0 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:24:30,281] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:48,037] INFO : Step 3200/50000 rank0 | Train Loss: 12.8622 | Train Loss Unweighted: 4.6798 | Train Loss Components: Classification: 0.0343 | Classification_binary: 4.1616 | Regression_cos_phi: 0.0103 | Regression_energy: 0.2233 | Regression_eta: 0.0180 | Regression_pt: 0.2233 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2918 | Classification_binary: 4.1616 | Regression_cos_phi: 0.6712 | Regression_energy: 2.4413 | Regression_eta: 0.8037 | Regression_pt: 2.6529 | Regression_sin_phi: 0.8397 | LR: 9.91e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.0962s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0145 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3107.0 | uss_MB: 2596.0 | child_rss_MB: 3237.9 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:24:48,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:05,931] INFO : Step 3300/50000 rank0 | Train Loss: 14.4116 | Train Loss Unweighted: 4.9272 | Train Loss Components: Classification: 0.0332 | Classification_binary: 4.2930 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2841 | Regression_eta: 0.0159 | Regression_pt: 0.2791 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2483 | Classification_binary: 4.2930 | Regression_cos_phi: 0.7250 | Regression_energy: 3.1056 | Regression_eta: 0.7105 | Regression_pt: 3.3168 | Regression_sin_phi: 1.0125 | LR: 9.90e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2383s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0149 | max_loss: 0.0555 | max_backward: 0.0331 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3108.0 | uss_MB: 2595.2 | child_rss_MB: 3264.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:25:06,146] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:23,832] INFO : Step 3400/50000 rank0 | Train Loss: 13.8794 | Train Loss Unweighted: 4.8292 | Train Loss Components: Classification: 0.0305 | Classification_binary: 4.2195 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2771 | Regression_eta: 0.0179 | Regression_pt: 0.2647 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1474 | Classification_binary: 4.2195 | Regression_cos_phi: 0.6631 | Regression_energy: 3.0294 | Regression_eta: 0.7988 | Regression_pt: 3.1457 | Regression_sin_phi: 0.8754 | LR: 9.90e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1968s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.2937 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3108.0 | uss_MB: 2596.6 | child_rss_MB: 3236.3 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:25:24,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:41,735] INFO : Step 3500/50000 rank0 | Train Loss: 13.1237 | Train Loss Unweighted: 4.5974 | Train Loss Components: Classification: 0.0309 | Classification_binary: 4.0277 | Regression_cos_phi: 0.0107 | Regression_energy: 0.2556 | Regression_eta: 0.0130 | Regression_pt: 0.2501 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1643 | Classification_binary: 4.0277 | Regression_cos_phi: 0.6955 | Regression_energy: 2.7943 | Regression_eta: 0.5778 | Regression_pt: 2.9718 | Regression_sin_phi: 0.8923 | LR: 9.89e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.0591s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0139 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.0983 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3108.0 | uss_MB: 2596.9 | child_rss_MB: 3237.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:25:41,949] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:59,473] INFO : Step 3600/50000 rank0 | Train Loss: 12.9620 | Train Loss Unweighted: 4.6313 | Train Loss Components: Classification: 0.0317 | Classification_binary: 4.1105 | Regression_cos_phi: 0.0097 | Regression_energy: 0.2267 | Regression_eta: 0.0229 | Regression_pt: 0.2201 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1951 | Classification_binary: 4.1105 | Regression_cos_phi: 0.6278 | Regression_energy: 2.4788 | Regression_eta: 1.0226 | Regression_pt: 2.6149 | Regression_sin_phi: 0.9124 | LR: 9.89e-04 | DataLoad Time: 0.1361s | Model Forward Time: 1.1387s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0029 | max_backward: 0.0905 | max_optimizer_step: 0.1073 | max_tensorboard_logging: 0.2735 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3108.0 | uss_MB: 2597.1 | child_rss_MB: 3235.9 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:25:59,685] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:17,266] INFO : Step 3700/50000 rank0 | Train Loss: 14.1425 | Train Loss Unweighted: 4.6245 | Train Loss Components: Classification: 0.0324 | Classification_binary: 4.0383 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2574 | Regression_eta: 0.0311 | Regression_pt: 0.2432 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2193 | Classification_binary: 4.0383 | Regression_cos_phi: 0.6564 | Regression_energy: 2.8140 | Regression_eta: 1.3850 | Regression_pt: 2.8897 | Regression_sin_phi: 1.1397 | LR: 9.88e-04 | DataLoad Time: 0.0759s | Model Forward Time: 1.2308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0542 | max_forward: 0.0167 | max_loss: 0.0031 | max_backward: 0.0343 | max_optimizer_step: 0.1022 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2967.5 | uss_MB: 2455.5 | child_rss_MB: 3232.5 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:26:17,479] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:34,812] INFO : Step 3800/50000 rank0 | Train Loss: 12.3815 | Train Loss Unweighted: 4.5512 | Train Loss Components: Classification: 0.0297 | Classification_binary: 4.0314 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2314 | Regression_eta: 0.0127 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1170 | Classification_binary: 4.0314 | Regression_cos_phi: 0.6206 | Regression_energy: 2.5300 | Regression_eta: 0.5665 | Regression_pt: 2.7079 | Regression_sin_phi: 0.8082 | LR: 9.87e-04 | DataLoad Time: 0.0381s | Model Forward Time: 1.0597s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1024 | max_tensorboard_logging: 0.2692 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 2967.5 | uss_MB: 2456.1 | child_rss_MB: 3229.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:26:35,026] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:52,550] INFO : Step 3900/50000 rank0 | Train Loss: 13.5429 | Train Loss Unweighted: 4.6169 | Train Loss Components: Classification: 0.0320 | Classification_binary: 4.0450 | Regression_cos_phi: 0.0095 | Regression_energy: 0.2535 | Regression_eta: 0.0220 | Regression_pt: 0.2441 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2033 | Classification_binary: 4.0450 | Regression_cos_phi: 0.6140 | Regression_energy: 2.7715 | Regression_eta: 0.9798 | Regression_pt: 2.9000 | Regression_sin_phi: 1.0294 | LR: 9.87e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1339s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0132 | max_loss: 0.0030 | max_backward: 0.1077 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2967.5 | uss_MB: 2456.5 | child_rss_MB: 3232.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:26:52,762] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:10,376] INFO : Step 4000/50000 rank0 | Train Loss: 12.6203 | Train Loss Unweighted: 4.5476 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.9891 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2513 | Regression_eta: 0.0106 | Regression_pt: 0.2507 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1068 | Classification_binary: 3.9891 | Regression_cos_phi: 0.5316 | Regression_energy: 2.7474 | Regression_eta: 0.4707 | Regression_pt: 2.9783 | Regression_sin_phi: 0.7963 | LR: 9.86e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0138 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1014 | max_tensorboard_logging: 0.2795 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 2969.5 | uss_MB: 2456.9 | child_rss_MB: 3231.8 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:10,590] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:10,618] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:27:10,699] INFO : Running validation on rank0 +rank0 [2026-10-28 11:27:10,699] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:27:10,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:11,794] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:27:11,937] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:12,069] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:12,143] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:12,212] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:12,280] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:12,349] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:12,418] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:12,520] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:12,642] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:12,764] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:12,836] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:12,905] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:12,972] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:13,040] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:13,109] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:13,221] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:13,228] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:13,231] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:13,234] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:13,237] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:13,238] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:13,240] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:13,243] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:13,246] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:13,247] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:13,250] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:13,253] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:13,256] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:13,259] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:13,261] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:13,262] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:19,160] INFO : VALIDATION | Step=4000/50000 | Train Loss=12.6203 | Valid Loss=12.6961 | Train Loss Unweighted=4.5476 | Valid Loss Unweighted=4.5506 | Stale=0 | ETA=169.8m +rank0 [2026-10-28 11:27:19,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:19,226] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:19,229] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:19,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:20,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:21,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:21,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:22,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:22,599] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:22,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:22,657] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:22,660] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:22,736] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:24,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:24,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:25,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:25,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:25,876] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:25,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:25,933] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:25,936] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:27:26,009] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:27,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:27,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:28,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:29,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:29,127] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:29,135] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:29,391] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:29,797] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:31,229] INFO : Plotted jets +rank0 [2026-10-28 11:27:32,303] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:32,334] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:32,566] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:33,370] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:34,756] INFO : Plotted jets +rank0 [2026-10-28 11:27:35,849] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:35,880] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:36,085] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:36,444] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:37,814] INFO : Plotted jets +rank0 [2026-10-28 11:27:38,857] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:56,313] INFO : Step 4100/50000 rank0 | Train Loss: 12.7175 | Train Loss Unweighted: 4.4683 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.9477 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2289 | Regression_eta: 0.0237 | Regression_pt: 0.2206 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0760 | Classification_binary: 3.9477 | Regression_cos_phi: 0.6020 | Regression_energy: 2.5024 | Regression_eta: 1.0548 | Regression_pt: 2.6209 | Regression_sin_phi: 0.9136 | LR: 9.85e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.0743s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0030 | max_backward: 0.0327 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3155.9 | uss_MB: 2644.2 | child_rss_MB: 3232.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:56,527] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:13,755] INFO : Step 4200/50000 rank0 | Train Loss: 12.7824 | Train Loss Unweighted: 4.5597 | Train Loss Components: Classification: 0.0313 | Classification_binary: 4.0394 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2326 | Regression_eta: 0.0169 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1772 | Classification_binary: 4.0394 | Regression_cos_phi: 0.7277 | Regression_energy: 2.5426 | Regression_eta: 0.7518 | Regression_pt: 2.5943 | Regression_sin_phi: 0.9494 | LR: 9.84e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.0889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3155.9 | uss_MB: 2644.4 | child_rss_MB: 3231.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:28:13,966] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:31,246] INFO : Step 4300/50000 rank0 | Train Loss: 12.1751 | Train Loss Unweighted: 4.4215 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.9301 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2166 | Regression_eta: 0.0202 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1190 | Classification_binary: 3.9301 | Regression_cos_phi: 0.5484 | Regression_energy: 2.3685 | Regression_eta: 0.8998 | Regression_pt: 2.4653 | Regression_sin_phi: 0.8440 | LR: 9.84e-04 | DataLoad Time: 0.0361s | Model Forward Time: 1.0886s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0145 | max_loss: 0.0037 | max_backward: 0.0331 | max_optimizer_step: 0.1052 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3155.9 | uss_MB: 2644.7 | child_rss_MB: 3226.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:28:31,459] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:48,655] INFO : Step 4400/50000 rank0 | Train Loss: 11.7903 | Train Loss Unweighted: 4.4421 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.9736 | Regression_cos_phi: 0.0100 | Regression_energy: 0.2058 | Regression_eta: 0.0112 | Regression_pt: 0.2042 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0110 | Classification_binary: 3.9736 | Regression_cos_phi: 0.6519 | Regression_energy: 2.2504 | Regression_eta: 0.4998 | Regression_pt: 2.4262 | Regression_sin_phi: 0.9774 | LR: 9.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2815s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0744 | max_forward: 0.0168 | max_loss: 0.0031 | max_backward: 0.0385 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.3834 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3155.9 | uss_MB: 2646.9 | child_rss_MB: 3227.2 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:28:48,869] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:06,118] INFO : Step 4500/50000 rank0 | Train Loss: 13.2269 | Train Loss Unweighted: 4.4148 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.8906 | Regression_cos_phi: 0.0135 | Regression_energy: 0.2314 | Regression_eta: 0.0296 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1301 | Classification_binary: 3.8906 | Regression_cos_phi: 0.8787 | Regression_energy: 2.5300 | Regression_eta: 1.3196 | Regression_pt: 2.4848 | Regression_sin_phi: 0.9931 | LR: 9.82e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2645s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0937 | max_forward: 0.0152 | max_loss: 0.0031 | max_backward: 0.0340 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3156.9 | uss_MB: 2645.4 | child_rss_MB: 3227.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:06,329] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:23,668] INFO : Step 4600/50000 rank0 | Train Loss: 11.8457 | Train Loss Unweighted: 4.4047 | Train Loss Components: Classification: 0.0306 | Classification_binary: 3.9345 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2053 | Regression_eta: 0.0134 | Regression_pt: 0.2018 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1520 | Classification_binary: 3.9345 | Regression_cos_phi: 0.6371 | Regression_energy: 2.2443 | Regression_eta: 0.5995 | Regression_pt: 2.3983 | Regression_sin_phi: 0.8800 | LR: 9.81e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0664s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.0998 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3156.9 | uss_MB: 2645.6 | child_rss_MB: 3230.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:29:23,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:40,732] INFO : Step 4700/50000 rank0 | Train Loss: 12.5723 | Train Loss Unweighted: 4.4591 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.9282 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2397 | Regression_eta: 0.0173 | Regression_pt: 0.2262 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1014 | Classification_binary: 3.9282 | Regression_cos_phi: 0.6256 | Regression_energy: 2.6203 | Regression_eta: 0.7707 | Regression_pt: 2.6877 | Regression_sin_phi: 0.8384 | LR: 9.81e-04 | DataLoad Time: 0.0924s | Model Forward Time: 1.2058s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.1194 | max_optimizer_step: 0.1052 | max_tensorboard_logging: 0.3005 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3046.5 | uss_MB: 2535.4 | child_rss_MB: 3230.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:40,945] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:57,852] INFO : Step 4800/50000 rank0 | Train Loss: 13.5812 | Train Loss Unweighted: 4.6317 | Train Loss Components: Classification: 0.0332 | Classification_binary: 4.0582 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2558 | Regression_eta: 0.0178 | Regression_pt: 0.2446 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.2507 | Classification_binary: 4.0582 | Regression_cos_phi: 0.6892 | Regression_energy: 2.7963 | Regression_eta: 0.7948 | Regression_pt: 2.9064 | Regression_sin_phi: 1.0856 | LR: 9.80e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.2148s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0307 | max_forward: 0.0151 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1034 | max_tensorboard_logging: 0.3824 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3048.5 | uss_MB: 2537.8 | child_rss_MB: 3227.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:58,066] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:15,587] INFO : Step 4900/50000 rank0 | Train Loss: 12.2599 | Train Loss Unweighted: 4.4027 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.8965 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2228 | Regression_eta: 0.0148 | Regression_pt: 0.2188 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1582 | Classification_binary: 3.8965 | Regression_cos_phi: 0.6606 | Regression_energy: 2.4357 | Regression_eta: 0.6595 | Regression_pt: 2.5992 | Regression_sin_phi: 0.8502 | LR: 9.79e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0145 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3048.5 | uss_MB: 2538.0 | child_rss_MB: 3223.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:30:15,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:32,832] INFO : Step 5000/50000 rank0 | Train Loss: 11.6048 | Train Loss Unweighted: 4.3258 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.8422 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2146 | Regression_eta: 0.0118 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1125 | Classification_binary: 3.8422 | Regression_cos_phi: 0.5225 | Regression_energy: 2.3466 | Regression_eta: 0.5254 | Regression_pt: 2.5163 | Regression_sin_phi: 0.7392 | LR: 9.78e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.0821s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1009 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3048.5 | uss_MB: 2538.1 | child_rss_MB: 3226.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:30:33,044] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 4% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:33,081] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:30:33,165] INFO : Running validation on rank0 +rank0 [2026-10-28 11:30:33,165] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:30:33,166] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:30:34,255] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:30:34,399] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:30:34,529] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:30:34,602] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:30:34,671] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:30:34,738] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:30:34,806] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:30:34,875] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:30:34,972] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:30:35,102] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:30:35,230] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:30:35,301] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:30:35,370] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:30:35,438] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:30:35,506] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:30:35,575] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:30:35,686] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:30:35,693] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:30:35,696] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:30:35,698] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:30:35,699] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:30:35,702] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:30:35,705] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:30:35,705] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:30:35,708] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:30:35,711] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:30:35,714] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:30:35,716] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:30:35,719] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:30:35,720] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:30:35,723] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:30:35,726] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:30:41,638] INFO : VALIDATION | Step=5000/50000 | Train Loss=11.6048 | Valid Loss=12.2601 | Train Loss Unweighted=4.3258 | Valid Loss Unweighted=4.4266 | Stale=0 | ETA=163.2m +rank0 [2026-10-28 11:30:41,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:41,705] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:30:41,709] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:30:41,856] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:43,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:44,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:44,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:45,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:45,467] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:30:45,470] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:45,529] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:30:45,532] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:45,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:47,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:47,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:48,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:48,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:48,840] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:48,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:48,898] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:30:48,901] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:30:48,978] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:50,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:50,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:51,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:51,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:51,983] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:52,002] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:30:52,258] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:30:52,666] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:54,109] INFO : Plotted jets +rank0 [2026-10-28 11:30:55,191] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:55,273] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:55,500] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:55,889] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:57,668] INFO : Plotted jets +rank0 [2026-10-28 11:30:58,765] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:58,798] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:30:59,002] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:30:59,365] INFO : Plotted number of elements +rank0 [2026-10-28 11:31:00,738] INFO : Plotted jets +rank0 [2026-10-28 11:31:01,787] INFO : Plotted jet ratio +rank0 [2026-10-28 11:31:19,406] INFO : Step 5100/50000 rank0 | Train Loss: 12.5418 | Train Loss Unweighted: 4.5385 | Train Loss Components: Classification: 0.0303 | Classification_binary: 4.0181 | Regression_cos_phi: 0.0073 | Regression_energy: 0.2299 | Regression_eta: 0.0230 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1406 | Classification_binary: 4.0181 | Regression_cos_phi: 0.4737 | Regression_energy: 2.5133 | Regression_eta: 1.0257 | Regression_pt: 2.6409 | Regression_sin_phi: 0.7296 | LR: 9.77e-04 | DataLoad Time: 0.0400s | Model Forward Time: 1.3004s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1080 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.3934 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3119.7 | uss_MB: 2611.2 | child_rss_MB: 3226.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:31:19,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 106.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:36,763] INFO : Step 5200/50000 rank0 | Train Loss: 12.4054 | Train Loss Unweighted: 4.5089 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.9835 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2364 | Regression_eta: 0.0145 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0491 | Classification_binary: 3.9835 | Regression_cos_phi: 0.5765 | Regression_energy: 2.5840 | Regression_eta: 0.6449 | Regression_pt: 2.7191 | Regression_sin_phi: 0.8484 | LR: 9.76e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2828s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0156 | max_loss: 0.1017 | max_backward: 0.0377 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.3877 | max_tensorboard_flush: 0.0038 | Memory: rss_MB: 3120.7 | uss_MB: 2613.3 | child_rss_MB: 3252.8 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:31:36,974] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:53,664] INFO : Step 5300/50000 rank0 | Train Loss: 11.5300 | Train Loss Unweighted: 4.2441 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.7826 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2038 | Regression_eta: 0.0150 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1241 | Classification_binary: 3.7826 | Regression_cos_phi: 0.6065 | Regression_energy: 2.2277 | Regression_eta: 0.6671 | Regression_pt: 2.3182 | Regression_sin_phi: 0.8039 | LR: 9.75e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2122s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0352 | max_forward: 0.0135 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1050 | max_tensorboard_logging: 0.3763 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3121.7 | uss_MB: 2612.9 | child_rss_MB: 3225.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:31:53,878] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:10,789] INFO : Step 5400/50000 rank0 | Train Loss: 12.0754 | Train Loss Unweighted: 4.1915 | Train Loss Components: Classification: 0.0285 | Classification_binary: 3.6673 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2335 | Regression_eta: 0.0150 | Regression_pt: 0.2299 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0710 | Classification_binary: 3.6673 | Regression_cos_phi: 0.5751 | Regression_energy: 2.5529 | Regression_eta: 0.6697 | Regression_pt: 2.7313 | Regression_sin_phi: 0.8081 | LR: 9.74e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.1601s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0140 | max_loss: 0.0029 | max_backward: 0.1367 | max_optimizer_step: 0.1043 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3123.7 | uss_MB: 2613.9 | child_rss_MB: 3227.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:32:11,002] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:27,848] INFO : Step 5500/50000 rank0 | Train Loss: 12.1733 | Train Loss Unweighted: 4.2507 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.7408 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2336 | Regression_eta: 0.0192 | Regression_pt: 0.2094 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1097 | Classification_binary: 3.7408 | Regression_cos_phi: 0.6591 | Regression_energy: 2.5539 | Regression_eta: 0.8562 | Regression_pt: 2.4876 | Regression_sin_phi: 0.7660 | LR: 9.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2267s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0579 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1029 | max_tensorboard_logging: 0.3867 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3124.7 | uss_MB: 2614.4 | child_rss_MB: 3222.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:32:28,062] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:44,706] INFO : Step 5600/50000 rank0 | Train Loss: 11.4903 | Train Loss Unweighted: 4.2869 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.8150 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2073 | Regression_eta: 0.0122 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0270 | Classification_binary: 3.8150 | Regression_cos_phi: 0.5647 | Regression_energy: 2.2669 | Regression_eta: 0.5431 | Regression_pt: 2.4704 | Regression_sin_phi: 0.8033 | LR: 9.72e-04 | DataLoad Time: 0.0628s | Model Forward Time: 1.1990s | Phase Times: max_pre_step_barrier: 0.0953 | max_batch_to_device: 0.0011 | max_forward: 0.0137 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1006 | max_tensorboard_logging: 0.3178 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3124.7 | uss_MB: 2615.0 | child_rss_MB: 3223.1 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:32:44,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:01,827] INFO : Step 5700/50000 rank0 | Train Loss: 12.0141 | Train Loss Unweighted: 4.3467 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.8520 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2208 | Regression_eta: 0.0154 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0490 | Classification_binary: 3.8520 | Regression_cos_phi: 0.6221 | Regression_energy: 2.4142 | Regression_eta: 0.6869 | Regression_pt: 2.5164 | Regression_sin_phi: 0.8734 | LR: 9.71e-04 | DataLoad Time: 0.0564s | Model Forward Time: 1.2105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0153 | max_loss: 0.0030 | max_backward: 0.1352 | max_optimizer_step: 0.1053 | max_tensorboard_logging: 0.3005 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3122.1 | uss_MB: 2612.4 | child_rss_MB: 3221.3 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:33:02,042] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:18,856] INFO : Step 5800/50000 rank0 | Train Loss: 12.0533 | Train Loss Unweighted: 4.3016 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.8058 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2201 | Regression_eta: 0.0159 | Regression_pt: 0.2123 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0453 | Classification_binary: 3.8058 | Regression_cos_phi: 0.6593 | Regression_energy: 2.4057 | Regression_eta: 0.7111 | Regression_pt: 2.5225 | Regression_sin_phi: 0.9036 | LR: 9.70e-04 | DataLoad Time: 0.0100s | Model Forward Time: 1.2515s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0576 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1025 | max_tensorboard_logging: 0.4215 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3122.1 | uss_MB: 2613.6 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:33:19,069] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:35,648] INFO : Step 5900/50000 rank0 | Train Loss: 11.9432 | Train Loss Unweighted: 4.2404 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.7576 | Regression_cos_phi: 0.0118 | Regression_energy: 0.2127 | Regression_eta: 0.0138 | Regression_pt: 0.2049 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1289 | Classification_binary: 3.7576 | Regression_cos_phi: 0.7637 | Regression_energy: 2.3249 | Regression_eta: 0.6153 | Regression_pt: 2.4343 | Regression_sin_phi: 0.9185 | LR: 9.69e-04 | DataLoad Time: 0.0441s | Model Forward Time: 1.2331s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0722 | max_forward: 0.0176 | max_loss: 0.0031 | max_backward: 0.0389 | max_optimizer_step: 0.1003 | max_tensorboard_logging: 0.3693 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3123.1 | uss_MB: 2615.9 | child_rss_MB: 3217.5 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:33:35,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:52,771] INFO : Step 6000/50000 rank0 | Train Loss: 11.6470 | Train Loss Unweighted: 4.3060 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.8349 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2074 | Regression_eta: 0.0118 | Regression_pt: 0.2035 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.1198 | Classification_binary: 3.8349 | Regression_cos_phi: 0.6014 | Regression_energy: 2.2679 | Regression_eta: 0.5273 | Regression_pt: 2.4183 | Regression_sin_phi: 0.8775 | LR: 9.68e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.1880s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0032 | max_backward: 0.0384 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3125.1 | uss_MB: 2617.3 | child_rss_MB: 3218.6 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:33:52,998] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:53,016] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:33:53,097] INFO : Running validation on rank0 +rank0 [2026-10-28 11:33:53,097] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:33:53,097] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:33:54,298] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:33:54,441] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:33:54,572] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:33:54,645] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:33:54,713] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:33:54,780] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:33:54,851] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:33:54,919] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:33:55,022] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:33:55,151] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:33:55,279] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:33:55,349] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:33:55,417] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:33:55,484] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:33:55,553] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:33:55,622] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:33:55,736] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:33:55,743] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:33:55,746] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:33:55,747] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:33:55,749] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:33:55,750] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:33:55,753] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:33:55,756] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:33:55,757] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:33:55,759] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:33:55,762] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:33:55,763] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:33:55,765] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:33:55,766] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:33:55,769] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:33:55,771] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:34:01,695] INFO : VALIDATION | Step=6000/50000 | Train Loss=11.6470 | Valid Loss=11.7646 | Train Loss Unweighted=4.3060 | Valid Loss Unweighted=4.2717 | Stale=0 | ETA=157.5m +rank0 [2026-10-28 11:34:01,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:01,763] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:34:01,766] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:01,848] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:03,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:03,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:04,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:05,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:05,244] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:05,247] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:05,302] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:34:05,314] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:05,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:06,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:07,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:07,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:08,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:08,565] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:34:08,568] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:08,621] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:34:08,624] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:34:08,701] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:10,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:10,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:11,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:11,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:11,700] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:34:11,723] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:11,983] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:12,388] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:13,825] INFO : Plotted jets +rank0 [2026-10-28 11:34:14,910] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:14,942] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:15,171] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:15,560] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:17,392] INFO : Plotted jets +rank0 [2026-10-28 11:34:18,496] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:18,531] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:34:18,737] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:34:19,099] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:20,474] INFO : Plotted jets +rank0 [2026-10-28 11:34:21,523] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:38,547] INFO : Step 6100/50000 rank0 | Train Loss: 11.5079 | Train Loss Unweighted: 4.2354 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.7590 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2107 | Regression_eta: 0.0130 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0237 | Classification_binary: 3.7590 | Regression_cos_phi: 0.5348 | Regression_energy: 2.3033 | Regression_eta: 0.5796 | Regression_pt: 2.4779 | Regression_sin_phi: 0.8296 | LR: 9.67e-04 | DataLoad Time: 0.0507s | Model Forward Time: 1.1045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.2942 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3194.6 | uss_MB: 2685.1 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:34:38,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:55,337] INFO : Step 6200/50000 rank0 | Train Loss: 12.1244 | Train Loss Unweighted: 4.2133 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.7298 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2126 | Regression_eta: 0.0170 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0543 | Classification_binary: 3.7298 | Regression_cos_phi: 0.7927 | Regression_energy: 2.3248 | Regression_eta: 0.7587 | Regression_pt: 2.4053 | Regression_sin_phi: 1.0588 | LR: 9.66e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1052s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3194.6 | uss_MB: 2686.5 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:55,550] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:12,180] INFO : Step 6300/50000 rank0 | Train Loss: 12.0051 | Train Loss Unweighted: 4.1784 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.6932 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2134 | Regression_eta: 0.0237 | Regression_pt: 0.2034 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9352 | Classification_binary: 3.6932 | Regression_cos_phi: 0.6728 | Regression_energy: 2.3334 | Regression_eta: 1.0574 | Regression_pt: 2.4167 | Regression_sin_phi: 0.8965 | LR: 9.65e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1401s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.3232 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3194.6 | uss_MB: 2686.9 | child_rss_MB: 3212.3 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:35:12,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:29,116] INFO : Step 6400/50000 rank0 | Train Loss: 12.0367 | Train Loss Unweighted: 4.2760 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.7806 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2195 | Regression_eta: 0.0153 | Regression_pt: 0.2133 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0028 | Classification_binary: 3.7806 | Regression_cos_phi: 0.6801 | Regression_energy: 2.4001 | Regression_eta: 0.6841 | Regression_pt: 2.5341 | Regression_sin_phi: 0.9548 | LR: 9.64e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.1421s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0033 | max_backward: 0.1012 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3194.6 | uss_MB: 2687.2 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:35:29,329] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:45,606] INFO : Step 6500/50000 rank0 | Train Loss: 11.4772 | Train Loss Unweighted: 4.1222 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.6574 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2052 | Regression_eta: 0.0184 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0450 | Classification_binary: 3.6574 | Regression_cos_phi: 0.5759 | Regression_energy: 2.2436 | Regression_eta: 0.8191 | Regression_pt: 2.3294 | Regression_sin_phi: 0.8068 | LR: 9.63e-04 | DataLoad Time: 0.0209s | Model Forward Time: 1.1951s | Phase Times: max_pre_step_barrier: 0.0952 | max_batch_to_device: 0.0011 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2929 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3194.6 | uss_MB: 2687.3 | child_rss_MB: 3214.1 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:35:45,820] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:02,397] INFO : Step 6600/50000 rank0 | Train Loss: 11.7705 | Train Loss Unweighted: 4.1216 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.6391 | Regression_cos_phi: 0.0091 | Regression_energy: 0.2146 | Regression_eta: 0.0192 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9663 | Classification_binary: 3.6391 | Regression_cos_phi: 0.5903 | Regression_energy: 2.3463 | Regression_eta: 0.8574 | Regression_pt: 2.4224 | Regression_sin_phi: 0.9486 | LR: 9.62e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2680s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0901 | max_forward: 0.0139 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1049 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 3194.6 | uss_MB: 2687.6 | child_rss_MB: 3214.0 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:36:02,610] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:19,207] INFO : Step 6700/50000 rank0 | Train Loss: 10.8334 | Train Loss Unweighted: 4.0381 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.5865 | Regression_cos_phi: 0.0080 | Regression_energy: 0.2014 | Regression_eta: 0.0117 | Regression_pt: 0.1976 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9584 | Classification_binary: 3.5865 | Regression_cos_phi: 0.5190 | Regression_energy: 2.2018 | Regression_eta: 0.5203 | Regression_pt: 2.3483 | Regression_sin_phi: 0.6990 | LR: 9.61e-04 | DataLoad Time: 0.1538s | Model Forward Time: 1.0918s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1007 | max_tensorboard_logging: 0.2810 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3194.6 | uss_MB: 2687.7 | child_rss_MB: 3212.4 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:36:19,421] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:35,592] INFO : Step 6800/50000 rank0 | Train Loss: 11.1651 | Train Loss Unweighted: 4.0179 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.5472 | Regression_cos_phi: 0.0078 | Regression_energy: 0.2105 | Regression_eta: 0.0112 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9353 | Classification_binary: 3.5472 | Regression_cos_phi: 0.5057 | Regression_energy: 2.3011 | Regression_eta: 0.5013 | Regression_pt: 2.4541 | Regression_sin_phi: 0.9204 | LR: 9.60e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1450s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0866 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3195.8 | uss_MB: 2686.8 | child_rss_MB: 3215.5 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:36:35,808] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:52,040] INFO : Step 6900/50000 rank0 | Train Loss: 11.5291 | Train Loss Unweighted: 4.1205 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.6531 | Regression_cos_phi: 0.0115 | Regression_energy: 0.2070 | Regression_eta: 0.0157 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0188 | Classification_binary: 3.6531 | Regression_cos_phi: 0.7435 | Regression_energy: 2.2632 | Regression_eta: 0.7006 | Regression_pt: 2.3493 | Regression_sin_phi: 0.8006 | LR: 9.58e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2153s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0354 | max_forward: 0.0165 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.3678 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3196.8 | uss_MB: 2690.2 | child_rss_MB: 3214.6 | allocated_MB: 210.7 | reserved_MB: 2738.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:36:52,252] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:08,724] INFO : Step 7000/50000 rank0 | Train Loss: 12.5052 | Train Loss Unweighted: 4.2046 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.6562 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2468 | Regression_eta: 0.0138 | Regression_pt: 0.2406 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9552 | Classification_binary: 3.6562 | Regression_cos_phi: 0.7365 | Regression_energy: 2.6980 | Regression_eta: 0.6133 | Regression_pt: 2.8592 | Regression_sin_phi: 0.9868 | LR: 9.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2394s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0602 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1053 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3196.8 | uss_MB: 2688.2 | child_rss_MB: 3210.5 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:37:08,938] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:08,974] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 11:37:09,154] INFO : Running validation on rank0 +rank0 [2026-10-28 11:37:09,155] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:37:09,155] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:37:10,274] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 11:37:10,418] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:37:10,550] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:37:10,622] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:37:10,691] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:37:10,758] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:37:10,827] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:37:10,895] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:37:10,997] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:37:11,118] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:37:11,240] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:37:11,312] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:37:11,380] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:37:11,448] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:37:11,516] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:37:11,585] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:37:11,703] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:37:11,710] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:37:11,713] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:37:11,716] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:37:11,717] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:37:11,719] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:37:11,720] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:37:11,723] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:37:11,724] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:37:11,727] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:37:11,729] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:37:11,730] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:37:11,733] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:37:11,735] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:37:11,738] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:37:11,739] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:37:17,766] INFO : VALIDATION | Step=7000/50000 | Train Loss=12.5052 | Valid Loss=11.4875 | Train Loss Unweighted=4.2046 | Valid Loss Unweighted=4.1305 | Stale=0 | ETA=152.0m +rank0 [2026-10-28 11:37:17,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:17,832] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:37:17,836] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:37:17,980] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:19,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:20,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:20,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:21,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:21,492] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:37:21,495] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:21,549] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:37:21,552] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:21,633] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:23,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:23,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:24,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:24,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:24,861] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:24,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:24,916] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:37:24,919] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:37:25,002] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:26,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:27,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:27,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:28,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:28,170] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:28,177] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:37:28,437] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:37:28,842] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:30,284] INFO : Plotted jets +rank0 [2026-10-28 11:37:31,363] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:31,395] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:31,627] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:32,018] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:33,848] INFO : Plotted jets +rank0 [2026-10-28 11:37:34,958] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:34,990] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 11:37:35,196] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 11:37:35,557] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:36,929] INFO : Plotted jets +rank0 [2026-10-28 11:37:37,979] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:54,650] INFO : Step 7100/50000 rank0 | Train Loss: 11.8157 | Train Loss Unweighted: 4.1552 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.6620 | Regression_cos_phi: 0.0091 | Regression_energy: 0.2219 | Regression_eta: 0.0194 | Regression_pt: 0.2101 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8794 | Classification_binary: 3.6620 | Regression_cos_phi: 0.5901 | Regression_energy: 2.4255 | Regression_eta: 0.8638 | Regression_pt: 2.4961 | Regression_sin_phi: 0.8988 | LR: 9.56e-04 | DataLoad Time: 0.1463s | Model Forward Time: 1.1802s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0695 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3409.6 | uss_MB: 2903.9 | child_rss_MB: 3219.1 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:37:54,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:11,025] INFO : Step 7200/50000 rank0 | Train Loss: 10.5667 | Train Loss Unweighted: 4.1051 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.6800 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1898 | Regression_eta: 0.0120 | Regression_pt: 0.1846 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8921 | Classification_binary: 3.6800 | Regression_cos_phi: 0.5112 | Regression_energy: 2.0752 | Regression_eta: 0.5333 | Regression_pt: 2.1939 | Regression_sin_phi: 0.6811 | LR: 9.55e-04 | DataLoad Time: 0.1264s | Model Forward Time: 1.3279s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1046 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1097 | max_tensorboard_logging: 0.3890 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3268.3 | uss_MB: 2763.5 | child_rss_MB: 3216.8 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:38:11,238] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:27,752] INFO : Step 7300/50000 rank0 | Train Loss: 11.2619 | Train Loss Unweighted: 4.1059 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.6374 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2073 | Regression_eta: 0.0130 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9568 | Classification_binary: 3.6374 | Regression_cos_phi: 0.5632 | Regression_energy: 2.2663 | Regression_eta: 0.5807 | Regression_pt: 2.4414 | Regression_sin_phi: 0.8162 | LR: 9.53e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2615s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0685 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1074 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3268.3 | uss_MB: 2764.1 | child_rss_MB: 3211.0 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:38:27,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:44,225] INFO : Step 7400/50000 rank0 | Train Loss: 11.8457 | Train Loss Unweighted: 4.0443 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.5303 | Regression_cos_phi: 0.0088 | Regression_energy: 0.2313 | Regression_eta: 0.0198 | Regression_pt: 0.2211 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9391 | Classification_binary: 3.5303 | Regression_cos_phi: 0.5685 | Regression_energy: 2.5286 | Regression_eta: 0.8819 | Regression_pt: 2.6269 | Regression_sin_phi: 0.7703 | LR: 9.52e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2088s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0030 | max_backward: 0.1410 | max_optimizer_step: 0.1088 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3268.3 | uss_MB: 2764.2 | child_rss_MB: 3212.6 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:38:44,438] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:00,502] INFO : Step 7500/50000 rank0 | Train Loss: 11.9213 | Train Loss Unweighted: 4.1695 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.6741 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2198 | Regression_eta: 0.0159 | Regression_pt: 0.2119 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 1.0455 | Classification_binary: 3.6741 | Regression_cos_phi: 0.6857 | Regression_energy: 2.4035 | Regression_eta: 0.7078 | Regression_pt: 2.5182 | Regression_sin_phi: 0.8865 | LR: 9.51e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2418s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0134 | max_loss: 0.0577 | max_backward: 0.0331 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3268.3 | uss_MB: 2763.3 | child_rss_MB: 3212.4 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:39:00,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:16,758] INFO : Step 7600/50000 rank0 | Train Loss: 12.9397 | Train Loss Unweighted: 4.2959 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.7543 | Regression_cos_phi: 0.0111 | Regression_energy: 0.2410 | Regression_eta: 0.0237 | Regression_pt: 0.2285 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9725 | Classification_binary: 3.7543 | Regression_cos_phi: 0.7189 | Regression_energy: 2.6352 | Regression_eta: 1.0549 | Regression_pt: 2.7145 | Regression_sin_phi: 1.0894 | LR: 9.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1026s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1021 | max_tensorboard_logging: 0.3074 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3268.3 | uss_MB: 2764.9 | child_rss_MB: 3210.2 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:39:16,969] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:33,154] INFO : Step 7700/50000 rank0 | Train Loss: 11.1650 | Train Loss Unweighted: 4.0008 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.5332 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2076 | Regression_eta: 0.0113 | Regression_pt: 0.2046 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9720 | Classification_binary: 3.5332 | Regression_cos_phi: 0.6023 | Regression_energy: 2.2698 | Regression_eta: 0.5059 | Regression_pt: 2.4310 | Regression_sin_phi: 0.8507 | LR: 9.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.1308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0504 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3268.3 | uss_MB: 2765.0 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:39:33,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:49,446] INFO : Step 7800/50000 rank0 | Train Loss: 11.3556 | Train Loss Unweighted: 4.0260 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.5479 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2115 | Regression_eta: 0.0140 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9623 | Classification_binary: 3.5479 | Regression_cos_phi: 0.5779 | Regression_energy: 2.3128 | Regression_eta: 0.6251 | Regression_pt: 2.4845 | Regression_sin_phi: 0.8450 | LR: 9.47e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2360s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0307 | max_backward: 0.0347 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.3873 | max_tensorboard_flush: 0.0013 | Memory: rss_MB: 3266.2 | uss_MB: 2763.3 | child_rss_MB: 3211.5 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:39:49,657] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:05,702] INFO : Step 7900/50000 rank0 | Train Loss: 11.4037 | Train Loss Unweighted: 4.0769 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.6122 | Regression_cos_phi: 0.0083 | Regression_energy: 0.2095 | Regression_eta: 0.0174 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9308 | Classification_binary: 3.6122 | Regression_cos_phi: 0.5407 | Regression_energy: 2.2901 | Regression_eta: 0.7768 | Regression_pt: 2.3154 | Regression_sin_phi: 0.9376 | LR: 9.46e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0318 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1068 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3266.2 | uss_MB: 2765.0 | child_rss_MB: 3211.4 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:40:05,914] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:21,826] INFO : Step 8000/50000 rank0 | Train Loss: 10.3619 | Train Loss Unweighted: 3.8947 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.4879 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1770 | Regression_eta: 0.0121 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9382 | Classification_binary: 3.4879 | Regression_cos_phi: 0.6574 | Regression_energy: 1.9353 | Regression_eta: 0.5396 | Regression_pt: 2.0783 | Regression_sin_phi: 0.7253 | LR: 9.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.0875s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 0.2800 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3266.2 | uss_MB: 2765.1 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:40:22,038] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:22,070] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 11:40:22,153] INFO : Running validation on rank0 +rank0 [2026-10-28 11:40:22,153] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:40:22,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:40:23,252] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 11:40:23,397] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:40:23,529] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:40:23,603] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:40:23,672] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:40:23,741] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:40:23,809] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:40:23,879] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:40:23,983] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:40:24,114] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:40:24,244] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:40:24,315] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:40:24,384] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:40:24,451] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:40:24,521] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:40:24,589] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:40:24,690] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:40:24,696] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:40:24,699] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:40:24,702] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:40:24,702] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:40:24,705] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:40:24,706] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:40:24,709] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:40:24,711] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:40:24,712] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:40:24,715] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:40:24,717] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:40:24,720] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:40:24,721] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:40:24,723] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:40:24,724] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:40:30,523] INFO : VALIDATION | Step=8000/50000 | Train Loss=10.3619 | Valid Loss=10.9081 | Train Loss Unweighted=3.8947 | Valid Loss Unweighted=3.9603 | Stale=0 | ETA=146.8m +rank0 [2026-10-28 11:40:30,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:30,590] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:40:30,593] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:40:30,737] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:32,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:32,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:33,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:33,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:33,966] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:33,969] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:34,024] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:40:34,027] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:34,109] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:35,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:36,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:36,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:37,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:37,380] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:37,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:37,437] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:40:37,440] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:40:37,497] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:38,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:39,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:39,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:40,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:40,525] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:40,532] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:40:40,791] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:40:41,195] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:42,637] INFO : Plotted jets +rank0 [2026-10-28 11:40:43,720] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:43,752] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:43,984] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:44,373] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:46,203] INFO : Plotted jets +rank0 [2026-10-28 11:40:47,316] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:47,348] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 11:40:47,551] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 11:40:47,911] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:49,279] INFO : Plotted jets +rank0 [2026-10-28 11:40:50,324] INFO : Plotted jet ratio +rank0 [2026-10-28 11:41:06,629] INFO : Step 8100/50000 rank0 | Train Loss: 10.9651 | Train Loss Unweighted: 3.8570 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.4188 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1917 | Regression_eta: 0.0159 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9515 | Classification_binary: 3.4188 | Regression_cos_phi: 0.5991 | Regression_energy: 2.0960 | Regression_eta: 0.7099 | Regression_pt: 2.2069 | Regression_sin_phi: 0.9830 | LR: 9.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2290s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0185 | max_forward: 0.0173 | max_loss: 0.0034 | max_backward: 0.0383 | max_optimizer_step: 0.1096 | max_tensorboard_logging: 0.3558 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3475.2 | uss_MB: 2977.7 | child_rss_MB: 3211.3 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:41:06,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:22,797] INFO : Step 8200/50000 rank0 | Train Loss: 10.7617 | Train Loss Unweighted: 3.9594 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.5239 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1925 | Regression_eta: 0.0171 | Regression_pt: 0.1874 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8300 | Classification_binary: 3.5239 | Regression_cos_phi: 0.5314 | Regression_energy: 2.1046 | Regression_eta: 0.7645 | Regression_pt: 2.2269 | Regression_sin_phi: 0.7804 | LR: 9.42e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0892s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1073 | max_tensorboard_logging: 0.2773 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3338.3 | uss_MB: 2838.0 | child_rss_MB: 3211.6 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:41:23,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:38,709] INFO : Step 8300/50000 rank0 | Train Loss: 10.9664 | Train Loss Unweighted: 3.9141 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.4512 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2079 | Regression_eta: 0.0116 | Regression_pt: 0.2020 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8419 | Classification_binary: 3.4512 | Regression_cos_phi: 0.7016 | Regression_energy: 2.2731 | Regression_eta: 0.5164 | Regression_pt: 2.4004 | Regression_sin_phi: 0.7819 | LR: 9.40e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2518s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0346 | max_backward: 0.0340 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0132 | Memory: rss_MB: 3339.3 | uss_MB: 2837.2 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:38,921] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:54,992] INFO : Step 8400/50000 rank0 | Train Loss: 10.8453 | Train Loss Unweighted: 4.0634 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.6276 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1939 | Regression_eta: 0.0134 | Regression_pt: 0.1876 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8877 | Classification_binary: 3.6276 | Regression_cos_phi: 0.5672 | Regression_energy: 2.1194 | Regression_eta: 0.5958 | Regression_pt: 2.2293 | Regression_sin_phi: 0.8185 | LR: 9.39e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0691 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3339.3 | uss_MB: 2838.6 | child_rss_MB: 3210.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:55,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:11,161] INFO : Step 8500/50000 rank0 | Train Loss: 11.0323 | Train Loss Unweighted: 3.9909 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.5437 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1970 | Regression_eta: 0.0126 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9783 | Classification_binary: 3.5437 | Regression_cos_phi: 0.5950 | Regression_energy: 2.1538 | Regression_eta: 0.5618 | Regression_pt: 2.2916 | Regression_sin_phi: 0.9081 | LR: 9.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.1425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0031 | max_backward: 0.0339 | max_optimizer_step: 0.1076 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3339.3 | uss_MB: 2838.7 | child_rss_MB: 3208.6 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:11,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:27,170] INFO : Step 8600/50000 rank0 | Train Loss: 10.6909 | Train Loss Unweighted: 3.9331 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.5060 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1862 | Regression_eta: 0.0132 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9486 | Classification_binary: 3.5060 | Regression_cos_phi: 0.5622 | Regression_energy: 2.0354 | Regression_eta: 0.5887 | Regression_pt: 2.1963 | Regression_sin_phi: 0.8538 | LR: 9.36e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1220s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3339.3 | uss_MB: 2838.9 | child_rss_MB: 3209.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:27,383] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:43,248] INFO : Step 8700/50000 rank0 | Train Loss: 11.3930 | Train Loss Unweighted: 3.9739 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.5127 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2046 | Regression_eta: 0.0185 | Regression_pt: 0.1931 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9336 | Classification_binary: 3.5127 | Regression_cos_phi: 0.7125 | Regression_energy: 2.2365 | Regression_eta: 0.8259 | Regression_pt: 2.2944 | Regression_sin_phi: 0.8774 | LR: 9.34e-04 | DataLoad Time: 0.1214s | Model Forward Time: 1.3086s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0552 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0156 | Memory: rss_MB: 3340.3 | uss_MB: 2839.0 | child_rss_MB: 3212.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:42:43,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:59,305] INFO : Step 8800/50000 rank0 | Train Loss: 10.3360 | Train Loss Unweighted: 3.9224 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.5130 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1800 | Regression_eta: 0.0117 | Regression_pt: 0.1756 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9857 | Classification_binary: 3.5130 | Regression_cos_phi: 0.5454 | Regression_energy: 1.9674 | Regression_eta: 0.5230 | Regression_pt: 2.0860 | Regression_sin_phi: 0.7154 | LR: 9.33e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1281s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1047 | max_tensorboard_logging: 0.3010 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3337.7 | uss_MB: 2837.6 | child_rss_MB: 3209.9 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:42:59,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:15,340] INFO : Step 8900/50000 rank0 | Train Loss: 10.7503 | Train Loss Unweighted: 3.9338 | Train Loss Components: Classification: 0.0233 | Classification_binary: 3.4975 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1933 | Regression_eta: 0.0122 | Regression_pt: 0.1889 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8784 | Classification_binary: 3.4975 | Regression_cos_phi: 0.6207 | Regression_energy: 2.1136 | Regression_eta: 0.5439 | Regression_pt: 2.2447 | Regression_sin_phi: 0.8516 | LR: 9.31e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1229s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.0327 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3337.7 | uss_MB: 2839.1 | child_rss_MB: 3212.2 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:43:15,551] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:30,995] INFO : Step 9000/50000 rank0 | Train Loss: 9.9140 | Train Loss Unweighted: 3.7386 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.3382 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1766 | Regression_eta: 0.0097 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9374 | Classification_binary: 3.3382 | Regression_cos_phi: 0.5213 | Regression_energy: 1.9306 | Regression_eta: 0.4330 | Regression_pt: 2.0659 | Regression_sin_phi: 0.6875 | LR: 9.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1891s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0155 | max_loss: 0.0033 | max_backward: 0.1403 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.2737 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3337.7 | uss_MB: 2839.2 | child_rss_MB: 3212.0 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:43:31,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:31,228] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 11:43:31,310] INFO : Running validation on rank0 +rank0 [2026-10-28 11:43:31,310] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:43:31,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:43:32,513] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 11:43:32,656] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:43:32,787] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:43:32,860] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:43:32,928] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:43:32,995] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:43:33,064] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:43:33,133] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:43:33,235] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:43:33,366] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:43:33,495] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:43:33,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:43:33,633] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:43:33,701] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:43:33,771] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:43:33,840] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:43:33,947] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:43:33,954] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:43:33,956] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:43:33,957] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:43:33,960] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:43:33,963] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:43:33,966] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:43:33,967] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:43:33,969] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:43:33,970] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:43:33,973] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:43:33,975] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:43:33,977] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:43:33,979] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:43:33,980] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:43:33,983] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:43:39,919] INFO : VALIDATION | Step=9000/50000 | Train Loss=9.9140 | Valid Loss=10.8381 | Train Loss Unweighted=3.7386 | Valid Loss Unweighted=3.9186 | Stale=0 | ETA=141.7m +rank0 [2026-10-28 11:43:39,929] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:39,986] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:43:39,989] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:43:40,138] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:41,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:42,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:42,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:43,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:43,405] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:43,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:43,462] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:43:43,465] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:43,613] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:45,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:45,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:46,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:46,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:46,867] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:46,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:46,924] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:43:46,926] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:43:46,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:48,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:48,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:49,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:50,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:50,169] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:50,176] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:43:50,432] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:43:50,842] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:52,277] INFO : Plotted jets +rank0 [2026-10-28 11:43:53,357] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:53,389] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:53,619] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:54,009] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:55,842] INFO : Plotted jets +rank0 [2026-10-28 11:43:56,943] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:56,975] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 11:43:57,177] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 11:43:57,540] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:58,913] INFO : Plotted jets +rank0 [2026-10-28 11:43:59,964] INFO : Plotted jet ratio +rank0 [2026-10-28 11:44:15,633] INFO : Step 9100/50000 rank0 | Train Loss: 11.1304 | Train Loss Unweighted: 3.9345 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.4702 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2063 | Regression_eta: 0.0154 | Regression_pt: 0.2009 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9015 | Classification_binary: 3.4702 | Regression_cos_phi: 0.5643 | Regression_energy: 2.2550 | Regression_eta: 0.6881 | Regression_pt: 2.3874 | Regression_sin_phi: 0.8638 | LR: 9.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1342s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3024 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3554.0 | uss_MB: 3054.9 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:44:15,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:31,613] INFO : Step 9200/50000 rank0 | Train Loss: 11.5859 | Train Loss Unweighted: 3.9661 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.4817 | Regression_cos_phi: 0.0102 | Regression_energy: 0.2151 | Regression_eta: 0.0215 | Regression_pt: 0.2052 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9089 | Classification_binary: 3.4817 | Regression_cos_phi: 0.6610 | Regression_energy: 2.3517 | Regression_eta: 0.9569 | Regression_pt: 2.4384 | Regression_sin_phi: 0.7873 | LR: 9.27e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1005s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3416.0 | uss_MB: 2915.7 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:44:31,829] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:47,718] INFO : Step 9300/50000 rank0 | Train Loss: 10.5747 | Train Loss Unweighted: 3.8222 | Train Loss Components: Classification: 0.0227 | Classification_binary: 3.3867 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1943 | Regression_eta: 0.0128 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8538 | Classification_binary: 3.3867 | Regression_cos_phi: 0.4826 | Regression_energy: 2.1241 | Regression_eta: 0.5729 | Regression_pt: 2.2416 | Regression_sin_phi: 0.9131 | LR: 9.25e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.1399s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.2858 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3416.0 | uss_MB: 2916.8 | child_rss_MB: 3212.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:44:47,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:03,887] INFO : Step 9400/50000 rank0 | Train Loss: 11.3247 | Train Loss Unweighted: 4.0123 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.5723 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1947 | Regression_eta: 0.0220 | Regression_pt: 0.1801 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8266 | Classification_binary: 3.5723 | Regression_cos_phi: 0.7184 | Regression_energy: 2.1287 | Regression_eta: 0.9833 | Regression_pt: 2.1405 | Regression_sin_phi: 0.9549 | LR: 9.24e-04 | DataLoad Time: 0.0300s | Model Forward Time: 1.2423s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 3416.0 | uss_MB: 2917.0 | child_rss_MB: 3212.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:45:04,113] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:19,698] INFO : Step 9500/50000 rank0 | Train Loss: 9.7562 | Train Loss Unweighted: 3.7062 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.3147 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1732 | Regression_eta: 0.0092 | Regression_pt: 0.1700 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8831 | Classification_binary: 3.3147 | Regression_cos_phi: 0.5149 | Regression_energy: 1.8937 | Regression_eta: 0.4100 | Regression_pt: 2.0204 | Regression_sin_phi: 0.7193 | LR: 9.22e-04 | DataLoad Time: 0.1256s | Model Forward Time: 1.1692s | Phase Times: max_pre_step_barrier: 0.0212 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1075 | max_tensorboard_logging: 0.3119 | max_tensorboard_flush: 0.0131 | Memory: rss_MB: 3416.0 | uss_MB: 2917.2 | child_rss_MB: 3212.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:19,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:35,959] INFO : Step 9600/50000 rank0 | Train Loss: 10.0621 | Train Loss Unweighted: 3.7616 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.3592 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1770 | Regression_eta: 0.0125 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9073 | Classification_binary: 3.3592 | Regression_cos_phi: 0.5241 | Regression_energy: 1.9346 | Regression_eta: 0.5593 | Regression_pt: 2.0576 | Regression_sin_phi: 0.7201 | LR: 9.21e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2404s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.2855 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3416.0 | uss_MB: 2917.3 | child_rss_MB: 3214.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:45:36,185] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:51,589] INFO : Step 9700/50000 rank0 | Train Loss: 9.8348 | Train Loss Unweighted: 3.7436 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.3450 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1767 | Regression_eta: 0.0110 | Regression_pt: 0.1735 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8816 | Classification_binary: 3.3450 | Regression_cos_phi: 0.4312 | Regression_energy: 1.9322 | Regression_eta: 0.4891 | Regression_pt: 2.0619 | Regression_sin_phi: 0.6940 | LR: 9.19e-04 | DataLoad Time: 0.0052s | Model Forward Time: 1.2646s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0352 | max_backward: 0.0341 | max_optimizer_step: 0.1077 | max_tensorboard_logging: 0.4034 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3416.0 | uss_MB: 2916.1 | child_rss_MB: 3207.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:45:51,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:07,376] INFO : Step 9800/50000 rank0 | Train Loss: 10.4143 | Train Loss Unweighted: 3.7808 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.3655 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1841 | Regression_eta: 0.0162 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8891 | Classification_binary: 3.3655 | Regression_cos_phi: 0.6076 | Regression_energy: 2.0126 | Regression_eta: 0.7218 | Regression_pt: 2.0691 | Regression_sin_phi: 0.7486 | LR: 9.17e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2033s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0329 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.3176 | max_tensorboard_flush: 0.0148 | Memory: rss_MB: 3414.4 | uss_MB: 2916.0 | child_rss_MB: 3214.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:46:07,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:23,134] INFO : Step 9900/50000 rank0 | Train Loss: 11.4992 | Train Loss Unweighted: 3.9741 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.5054 | Regression_cos_phi: 0.0088 | Regression_energy: 0.2074 | Regression_eta: 0.0143 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9078 | Classification_binary: 3.5054 | Regression_cos_phi: 0.5713 | Regression_energy: 2.2672 | Regression_eta: 0.6392 | Regression_pt: 2.3915 | Regression_sin_phi: 1.2168 | LR: 9.16e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.3277s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0990 | max_loss: 0.0032 | max_backward: 0.0387 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.3895 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3415.4 | uss_MB: 2919.8 | child_rss_MB: 3216.3 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:46:23,348] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:38,823] INFO : Step 10000/50000 rank0 | Train Loss: 11.1498 | Train Loss Unweighted: 3.9044 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.4267 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2157 | Regression_eta: 0.0131 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8916 | Classification_binary: 3.4267 | Regression_cos_phi: 0.6378 | Regression_energy: 2.3583 | Regression_eta: 0.5847 | Regression_pt: 2.4597 | Regression_sin_phi: 0.7909 | LR: 9.14e-04 | DataLoad Time: 0.0397s | Model Forward Time: 1.1513s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0178 | max_loss: 0.0030 | max_backward: 0.0391 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3415.4 | uss_MB: 2920.4 | child_rss_MB: 3213.2 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:46:39,049] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:39,069] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 11:46:39,152] INFO : Running validation on rank0 +rank0 [2026-10-28 11:46:39,152] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:46:39,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:46:40,371] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 11:46:40,517] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:46:40,647] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:46:40,720] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:46:40,788] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:46:40,856] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:46:40,926] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:46:40,995] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:46:41,097] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:46:41,218] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:46:41,339] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:46:41,409] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:46:41,478] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:46:41,545] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:46:41,614] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:46:41,683] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:46:41,786] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:46:41,792] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:46:41,795] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:46:41,796] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:46:41,798] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:46:41,799] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:46:41,802] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:46:41,803] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:46:41,805] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:46:41,808] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:46:41,809] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:46:41,812] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:46:41,814] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:46:41,815] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:46:41,818] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:46:41,820] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:46:47,729] INFO : VALIDATION | Step=10000/50000 | Train Loss=11.1498 | Valid Loss=10.5914 | Train Loss Unweighted=3.9044 | Valid Loss Unweighted=3.8078 | Stale=0 | ETA=137.0m +rank0 [2026-10-28 11:46:47,737] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:47,793] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:46:47,797] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:46:47,945] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:49,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:50,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:50,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:51,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:51,337] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:46:51,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:51,396] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:46:51,399] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:51,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:52,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:53,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:53,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:54,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:54,707] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:54,710] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:54,765] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:46:55,040] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:46:55,188] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:56,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:57,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:57,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:58,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:58,321] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:58,328] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:46:58,588] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:46:58,991] INFO : Plotted number of elements +rank0 [2026-10-28 11:47:00,424] INFO : Plotted jets +rank0 [2026-10-28 11:47:01,501] INFO : Plotted jet ratio +rank0 [2026-10-28 11:47:01,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:47:01,763] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:47:02,154] INFO : Plotted number of elements +rank0 [2026-10-28 11:47:03,958] INFO : Plotted jets +rank0 [2026-10-28 11:47:05,061] INFO : Plotted jet ratio +rank0 [2026-10-28 11:47:05,093] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 11:47:05,297] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 11:47:05,659] INFO : Plotted number of elements +rank0 [2026-10-28 11:47:07,035] INFO : Plotted jets +rank0 [2026-10-28 11:47:08,081] INFO : Plotted jet ratio +rank0 [2026-10-28 11:47:23,759] INFO : Step 10100/50000 rank0 | Train Loss: 10.5975 | Train Loss Unweighted: 3.8344 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.4326 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1719 | Regression_eta: 0.0183 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9911 | Classification_binary: 3.4326 | Regression_cos_phi: 0.7014 | Regression_energy: 1.8798 | Regression_eta: 0.8148 | Regression_pt: 1.9727 | Regression_sin_phi: 0.8051 | LR: 9.12e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.1551s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0134 | Memory: rss_MB: 3627.7 | uss_MB: 3132.6 | child_rss_MB: 3208.0 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:47:23,975] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:39,339] INFO : Step 10200/50000 rank0 | Train Loss: 10.6366 | Train Loss Unweighted: 3.7614 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.3285 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1874 | Regression_eta: 0.0170 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9376 | Classification_binary: 3.3285 | Regression_cos_phi: 0.5613 | Regression_energy: 2.0486 | Regression_eta: 0.7587 | Regression_pt: 2.2188 | Regression_sin_phi: 0.7833 | LR: 9.11e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2613s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.3045 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3487.2 | uss_MB: 2991.2 | child_rss_MB: 3209.6 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:47:39,566] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:55,160] INFO : Step 10300/50000 rank0 | Train Loss: 11.8374 | Train Loss Unweighted: 3.9108 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.4172 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2166 | Regression_eta: 0.0249 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9414 | Classification_binary: 3.4172 | Regression_cos_phi: 0.6038 | Regression_energy: 2.3678 | Regression_eta: 1.1114 | Regression_pt: 2.4725 | Regression_sin_phi: 0.9234 | LR: 9.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1286s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1075 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3487.2 | uss_MB: 2992.0 | child_rss_MB: 3211.1 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:47:55,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:10,646] INFO : Step 10400/50000 rank0 | Train Loss: 10.2782 | Train Loss Unweighted: 3.8071 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3794 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1907 | Regression_eta: 0.0098 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9569 | Classification_binary: 3.3794 | Regression_cos_phi: 0.4522 | Regression_energy: 2.0848 | Regression_eta: 0.4363 | Regression_pt: 2.2204 | Regression_sin_phi: 0.7481 | LR: 9.07e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1127 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3487.2 | uss_MB: 2992.4 | child_rss_MB: 3215.8 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:48:10,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:25,985] INFO : Step 10500/50000 rank0 | Train Loss: 10.5189 | Train Loss Unweighted: 3.8277 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3966 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1905 | Regression_eta: 0.0120 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9568 | Classification_binary: 3.3966 | Regression_cos_phi: 0.5339 | Regression_energy: 2.0830 | Regression_eta: 0.5351 | Regression_pt: 2.2164 | Regression_sin_phi: 0.7972 | LR: 9.06e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1257s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0031 | max_backward: 0.0334 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3488.2 | uss_MB: 2993.0 | child_rss_MB: 3219.0 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:26,198] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:41,751] INFO : Step 10600/50000 rank0 | Train Loss: 11.2211 | Train Loss Unweighted: 3.8899 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.4175 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2106 | Regression_eta: 0.0170 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9227 | Classification_binary: 3.4175 | Regression_cos_phi: 0.5778 | Regression_energy: 2.3025 | Regression_eta: 0.7601 | Regression_pt: 2.4059 | Regression_sin_phi: 0.8346 | LR: 9.04e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1689s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1349 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3488.2 | uss_MB: 2993.1 | child_rss_MB: 3218.4 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:41,989] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:57,629] INFO : Step 10700/50000 rank0 | Train Loss: 11.0318 | Train Loss Unweighted: 3.8420 | Train Loss Components: Classification: 0.0233 | Classification_binary: 3.4021 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1948 | Regression_eta: 0.0204 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8778 | Classification_binary: 3.4021 | Regression_cos_phi: 0.5850 | Regression_energy: 2.1295 | Regression_eta: 0.9079 | Regression_pt: 2.1642 | Regression_sin_phi: 0.9652 | LR: 9.02e-04 | DataLoad Time: 0.0892s | Model Forward Time: 1.3055s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1084 | max_forward: 0.0139 | max_loss: 0.0035 | max_backward: 0.0332 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.3691 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3488.2 | uss_MB: 2993.1 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:48:57,843] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:13,134] INFO : Step 10800/50000 rank0 | Train Loss: 10.3468 | Train Loss Unweighted: 3.6965 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.2634 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1932 | Regression_eta: 0.0099 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9966 | Classification_binary: 3.2634 | Regression_cos_phi: 0.5483 | Regression_energy: 2.1120 | Regression_eta: 0.4398 | Regression_pt: 2.2227 | Regression_sin_phi: 0.7640 | LR: 9.00e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1744s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3300 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 3488.2 | uss_MB: 2993.2 | child_rss_MB: 3218.5 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:49:13,349] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:28,433] INFO : Step 10900/50000 rank0 | Train Loss: 9.8999 | Train Loss Unweighted: 3.7625 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.3598 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1800 | Regression_eta: 0.0105 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8417 | Classification_binary: 3.3598 | Regression_cos_phi: 0.4500 | Regression_energy: 1.9678 | Regression_eta: 0.4683 | Regression_pt: 2.0821 | Regression_sin_phi: 0.7303 | LR: 8.99e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1345s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0030 | max_backward: 0.0328 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.2894 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3488.3 | uss_MB: 2992.6 | child_rss_MB: 3217.7 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:49:28,645] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:43,755] INFO : Step 11000/50000 rank0 | Train Loss: 9.9850 | Train Loss Unweighted: 3.6916 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.2872 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1817 | Regression_eta: 0.0143 | Regression_pt: 0.1711 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8452 | Classification_binary: 3.2872 | Regression_cos_phi: 0.4527 | Regression_energy: 1.9870 | Regression_eta: 0.6369 | Regression_pt: 2.0329 | Regression_sin_phi: 0.7432 | LR: 8.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2486s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0288 | max_forward: 0.0134 | max_loss: 0.0032 | max_backward: 0.0320 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3852 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3488.3 | uss_MB: 2992.7 | child_rss_MB: 3220.0 | allocated_MB: 207.0 | reserved_MB: 2794.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:49:43,967] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:44,022] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 11:49:44,104] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:44,104] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:44,104] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:49:45,199] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 11:49:45,344] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:49:45,475] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:49:45,549] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:49:45,617] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:49:45,683] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:49:45,752] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:49:45,820] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:49:45,925] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:49:46,056] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:49:46,185] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:49:46,257] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:49:46,325] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:49:46,392] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:49:46,461] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:49:46,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:49:46,634] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:49:46,641] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:49:46,643] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:49:46,645] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:49:46,647] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:49:46,650] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:49:46,651] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:49:46,653] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:49:46,656] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:49:46,657] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:49:46,659] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:49:46,662] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:49:46,663] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:49:46,665] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:49:46,666] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:49:46,669] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:49:52,714] INFO : VALIDATION | Step=11000/50000 | Train Loss=9.9850 | Valid Loss=10.3084 | Train Loss Unweighted=3.6916 | Valid Loss Unweighted=3.7173 | Stale=0 | ETA=132.3m +rank0 [2026-10-28 11:49:52,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:52,781] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:49:52,784] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:52,936] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:54,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:55,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:55,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:56,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:56,465] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:49:56,469] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:56,524] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:49:56,527] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:56,586] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:58,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:58,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:59,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:59,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:59,809] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:49:59,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:59,867] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:49:59,870] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:49:59,957] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:50:01,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:50:01,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:50:02,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:50:02,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:50:02,969] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:50:02,976] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:50:03,230] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:50:03,632] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:05,070] INFO : Plotted jets +rank0 [2026-10-28 11:50:06,143] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:06,175] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:50:06,401] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:50:06,791] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:08,602] INFO : Plotted jets +rank0 [2026-10-28 11:50:09,700] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:09,733] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:09,939] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:10,304] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:11,681] INFO : Plotted jets +rank0 [2026-10-28 11:50:12,720] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:27,959] INFO : Step 11100/50000 rank0 | Train Loss: 10.5068 | Train Loss Unweighted: 3.6474 | Train Loss Components: Classification: 0.0230 | Classification_binary: 3.2021 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1993 | Regression_eta: 0.0143 | Regression_pt: 0.1918 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8640 | Classification_binary: 3.2021 | Regression_cos_phi: 0.5654 | Regression_energy: 2.1784 | Regression_eta: 0.6369 | Regression_pt: 2.2789 | Regression_sin_phi: 0.7811 | LR: 8.95e-04 | DataLoad Time: 0.0879s | Model Forward Time: 1.2050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0662 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3696.7 | uss_MB: 3203.7 | child_rss_MB: 3233.4 | allocated_MB: 210.7 | reserved_MB: 2746.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:50:28,170] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:43,331] INFO : Step 11200/50000 rank0 | Train Loss: 10.8775 | Train Loss Unweighted: 3.7238 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.2707 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2010 | Regression_eta: 0.0194 | Regression_pt: 0.1931 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8358 | Classification_binary: 3.2707 | Regression_cos_phi: 0.5255 | Regression_energy: 2.1976 | Regression_eta: 0.8632 | Regression_pt: 2.2941 | Regression_sin_phi: 0.8905 | LR: 8.93e-04 | DataLoad Time: 0.0187s | Model Forward Time: 1.2969s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.1351 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3048 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3696.7 | uss_MB: 3201.7 | child_rss_MB: 3215.4 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:50:43,542] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:58,368] INFO : Step 11300/50000 rank0 | Train Loss: 9.8796 | Train Loss Unweighted: 3.6881 | Train Loss Components: Classification: 0.0246 | Classification_binary: 3.2907 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1756 | Regression_eta: 0.0105 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9264 | Classification_binary: 3.2907 | Regression_cos_phi: 0.5450 | Regression_energy: 1.9199 | Regression_eta: 0.4703 | Regression_pt: 2.0303 | Regression_sin_phi: 0.6969 | LR: 8.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0427 | max_forward: 0.0137 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3675 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3558.3 | uss_MB: 3064.8 | child_rss_MB: 3218.2 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:50:58,581] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:13,373] INFO : Step 11400/50000 rank0 | Train Loss: 11.3860 | Train Loss Unweighted: 3.8579 | Train Loss Components: Classification: 0.0244 | Classification_binary: 3.3546 | Regression_cos_phi: 0.0091 | Regression_energy: 0.2282 | Regression_eta: 0.0133 | Regression_pt: 0.2196 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9202 | Classification_binary: 3.3546 | Regression_cos_phi: 0.5917 | Regression_energy: 2.4945 | Regression_eta: 0.5925 | Regression_pt: 2.6091 | Regression_sin_phi: 0.8235 | LR: 8.89e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2286s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.1205 | max_optimizer_step: 0.1127 | max_tensorboard_logging: 0.2843 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3559.3 | uss_MB: 3064.8 | child_rss_MB: 3213.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:51:13,586] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:28,663] INFO : Step 11500/50000 rank0 | Train Loss: 10.6484 | Train Loss Unweighted: 3.8270 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.4011 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1854 | Regression_eta: 0.0169 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9015 | Classification_binary: 3.4011 | Regression_cos_phi: 0.5417 | Regression_energy: 2.0271 | Regression_eta: 0.7526 | Regression_pt: 2.1659 | Regression_sin_phi: 0.8585 | LR: 8.88e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2278s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3559.3 | uss_MB: 3065.2 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:51:28,886] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:44,116] INFO : Step 11600/50000 rank0 | Train Loss: 10.6840 | Train Loss Unweighted: 3.8588 | Train Loss Components: Classification: 0.0227 | Classification_binary: 3.4082 | Regression_cos_phi: 0.0080 | Regression_energy: 0.2044 | Regression_eta: 0.0126 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8544 | Classification_binary: 3.4082 | Regression_cos_phi: 0.5214 | Regression_energy: 2.2351 | Regression_eta: 0.5628 | Regression_pt: 2.3115 | Regression_sin_phi: 0.7907 | LR: 8.86e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2253s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.1393 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3559.3 | uss_MB: 3065.5 | child_rss_MB: 3217.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:51:44,328] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:59,186] INFO : Step 11700/50000 rank0 | Train Loss: 10.0076 | Train Loss Unweighted: 3.6371 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.2459 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1722 | Regression_eta: 0.0177 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8991 | Classification_binary: 3.2459 | Regression_cos_phi: 0.5693 | Regression_energy: 1.8823 | Regression_eta: 0.7916 | Regression_pt: 1.9147 | Regression_sin_phi: 0.7049 | LR: 8.84e-04 | DataLoad Time: 0.0539s | Model Forward Time: 1.3118s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0644 | max_forward: 0.0167 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.3734 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3560.3 | uss_MB: 3065.5 | child_rss_MB: 3215.5 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:51:59,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:14,360] INFO : Step 11800/50000 rank0 | Train Loss: 10.2664 | Train Loss Unweighted: 3.6090 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.1746 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1944 | Regression_eta: 0.0129 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8434 | Classification_binary: 3.1746 | Regression_cos_phi: 0.6067 | Regression_energy: 2.1255 | Regression_eta: 0.5753 | Regression_pt: 2.2320 | Regression_sin_phi: 0.7088 | LR: 8.82e-04 | DataLoad Time: 0.0267s | Model Forward Time: 1.3599s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0188 | max_loss: 0.0807 | max_backward: 0.0393 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 3560.3 | uss_MB: 3066.7 | child_rss_MB: 3213.9 | allocated_MB: 210.7 | reserved_MB: 2746.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 11:52:14,571] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:29,376] INFO : Step 11900/50000 rank0 | Train Loss: 10.2338 | Train Loss Unweighted: 3.6931 | Train Loss Components: Classification: 0.0238 | Classification_binary: 3.3072 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1657 | Regression_eta: 0.0208 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8975 | Classification_binary: 3.3072 | Regression_cos_phi: 0.5901 | Regression_energy: 1.8115 | Regression_eta: 0.9293 | Regression_pt: 1.8737 | Regression_sin_phi: 0.8245 | LR: 8.80e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2321s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3559.4 | uss_MB: 3064.9 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:52:29,598] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:44,478] INFO : Step 12000/50000 rank0 | Train Loss: 10.6141 | Train Loss Unweighted: 3.7320 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.3009 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1914 | Regression_eta: 0.0163 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8596 | Classification_binary: 3.3009 | Regression_cos_phi: 0.5432 | Regression_energy: 2.0924 | Regression_eta: 0.7288 | Regression_pt: 2.1679 | Regression_sin_phi: 0.9214 | LR: 8.78e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0888 | max_forward: 0.0139 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.1056 | max_tensorboard_logging: 0.3638 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3559.4 | uss_MB: 3065.4 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:52:44,693] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:44,781] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 11:52:44,865] INFO : Running validation on rank0 +rank0 [2026-10-28 11:52:44,866] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:52:44,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:52:45,971] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 11:52:46,116] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:52:46,247] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:52:46,320] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:52:46,388] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:52:46,455] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:52:46,523] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:52:46,592] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:52:46,694] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:52:46,825] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:52:46,954] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:52:47,026] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:52:47,095] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:52:47,162] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:52:47,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:52:47,299] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:52:47,402] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:52:47,408] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:52:47,411] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:52:47,412] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:52:47,414] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:52:47,417] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:52:47,418] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:52:47,421] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:52:47,423] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:52:47,424] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:52:47,427] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:52:47,428] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:52:47,430] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:52:47,433] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:52:47,434] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:52:47,437] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:52:53,497] INFO : VALIDATION | Step=12000/50000 | Train Loss=10.6141 | Valid Loss=10.2467 | Train Loss Unweighted=3.7320 | Valid Loss Unweighted=3.7054 | Stale=0 | ETA=127.7m +rank0 [2026-10-28 11:52:53,506] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:53,563] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:52:53,567] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:52:53,718] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:55,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:55,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:56,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:56,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:56,967] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:52:56,970] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:57,025] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:52:57,028] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:57,112] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:58,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:59,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:59,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:53:00,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:53:00,361] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:53:00,364] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:53:00,417] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:53:00,420] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:53:00,571] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:53:01,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:53:02,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:53:03,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:53:03,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:53:03,959] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:53:03,966] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:53:04,226] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:53:04,632] INFO : Plotted number of elements +rank0 [2026-10-28 11:53:06,063] INFO : Plotted jets +rank0 [2026-10-28 11:53:07,143] INFO : Plotted jet ratio +rank0 [2026-10-28 11:53:07,174] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:53:07,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:53:07,796] INFO : Plotted number of elements +rank0 [2026-10-28 11:53:09,604] INFO : Plotted jets +rank0 [2026-10-28 11:53:10,712] INFO : Plotted jet ratio +rank0 [2026-10-28 11:53:10,744] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 11:53:10,954] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 11:53:11,316] INFO : Plotted number of elements +rank0 [2026-10-28 11:53:12,689] INFO : Plotted jets +rank0 [2026-10-28 11:53:13,735] INFO : Plotted jet ratio +rank0 [2026-10-28 11:53:28,716] INFO : Step 12100/50000 rank0 | Train Loss: 10.0259 | Train Loss Unweighted: 3.6415 | Train Loss Components: Classification: 0.0223 | Classification_binary: 3.2664 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1589 | Regression_eta: 0.0154 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8393 | Classification_binary: 3.2664 | Regression_cos_phi: 0.7729 | Regression_energy: 1.7370 | Regression_eta: 0.6847 | Regression_pt: 1.8731 | Regression_sin_phi: 0.8524 | LR: 8.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.1225 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.2792 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3755.4 | uss_MB: 3263.0 | child_rss_MB: 3210.7 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:28,928] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:44,007] INFO : Step 12200/50000 rank0 | Train Loss: 10.5149 | Train Loss Unweighted: 3.6655 | Train Loss Components: Classification: 0.0210 | Classification_binary: 3.2436 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1875 | Regression_eta: 0.0227 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7895 | Classification_binary: 3.2436 | Regression_cos_phi: 0.4939 | Regression_energy: 2.0499 | Regression_eta: 1.0121 | Regression_pt: 2.0678 | Regression_sin_phi: 0.8579 | LR: 8.74e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2172s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3755.4 | uss_MB: 3263.0 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:44,231] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:58,771] INFO : Step 12300/50000 rank0 | Train Loss: 10.1388 | Train Loss Unweighted: 3.6154 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.2029 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1822 | Regression_eta: 0.0128 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9409 | Classification_binary: 3.2029 | Regression_cos_phi: 0.4710 | Regression_energy: 1.9914 | Regression_eta: 0.5719 | Regression_pt: 2.0924 | Regression_sin_phi: 0.8683 | LR: 8.72e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2529s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0169 | max_loss: 0.0039 | max_backward: 0.1459 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2839 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3634.2 | uss_MB: 3144.9 | child_rss_MB: 3210.8 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:53:58,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:14,067] INFO : Step 12400/50000 rank0 | Train Loss: 10.0482 | Train Loss Unweighted: 3.6144 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.2086 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1781 | Regression_eta: 0.0153 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9443 | Classification_binary: 3.2086 | Regression_cos_phi: 0.5221 | Regression_energy: 1.9472 | Regression_eta: 0.6831 | Regression_pt: 2.0423 | Regression_sin_phi: 0.7008 | LR: 8.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2463s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0033 | max_backward: 0.0381 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3634.2 | uss_MB: 3145.3 | child_rss_MB: 3208.6 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:54:14,291] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:29,113] INFO : Step 12500/50000 rank0 | Train Loss: 10.0764 | Train Loss Unweighted: 3.6261 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.1989 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1906 | Regression_eta: 0.0105 | Regression_pt: 0.1880 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8570 | Classification_binary: 3.1989 | Regression_cos_phi: 0.4602 | Regression_energy: 2.0839 | Regression_eta: 0.4688 | Regression_pt: 2.2334 | Regression_sin_phi: 0.7742 | LR: 8.68e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0481 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1112 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3634.2 | uss_MB: 3143.8 | child_rss_MB: 3207.2 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:54:29,324] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:43,996] INFO : Step 12600/50000 rank0 | Train Loss: 9.7628 | Train Loss Unweighted: 3.6329 | Train Loss Components: Classification: 0.0239 | Classification_binary: 3.2504 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1649 | Regression_eta: 0.0133 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9003 | Classification_binary: 3.2504 | Regression_cos_phi: 0.5113 | Regression_energy: 1.8030 | Regression_eta: 0.5925 | Regression_pt: 1.9563 | Regression_sin_phi: 0.7491 | LR: 8.66e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1328s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.2893 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3634.2 | uss_MB: 3143.9 | child_rss_MB: 3207.2 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:44,211] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:58,831] INFO : Step 12700/50000 rank0 | Train Loss: 9.7425 | Train Loss Unweighted: 3.5927 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.2049 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1699 | Regression_eta: 0.0137 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9389 | Classification_binary: 3.2049 | Regression_cos_phi: 0.4641 | Regression_energy: 1.8576 | Regression_eta: 0.6103 | Regression_pt: 1.9548 | Regression_sin_phi: 0.7118 | LR: 8.64e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2236s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0159 | max_loss: 0.0030 | max_backward: 0.0338 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3634.2 | uss_MB: 3144.0 | child_rss_MB: 3205.1 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:59,063] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:13,503] INFO : Step 12800/50000 rank0 | Train Loss: 10.2311 | Train Loss Unweighted: 3.5411 | Train Loss Components: Classification: 0.0240 | Classification_binary: 3.1167 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1891 | Regression_eta: 0.0132 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9018 | Classification_binary: 3.1167 | Regression_cos_phi: 0.6234 | Regression_energy: 2.0669 | Regression_eta: 0.5880 | Regression_pt: 2.1415 | Regression_sin_phi: 0.7929 | LR: 8.62e-04 | DataLoad Time: 0.0097s | Model Forward Time: 1.1792s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0135 | max_loss: 0.0030 | max_backward: 0.0331 | max_optimizer_step: 0.1449 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3634.2 | uss_MB: 3144.2 | child_rss_MB: 3203.9 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:55:13,732] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:28,261] INFO : Step 12900/50000 rank0 | Train Loss: 10.0436 | Train Loss Unweighted: 3.5462 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.1463 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1735 | Regression_eta: 0.0159 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8863 | Classification_binary: 3.1463 | Regression_cos_phi: 0.5189 | Regression_energy: 1.8970 | Regression_eta: 0.7095 | Regression_pt: 2.0172 | Regression_sin_phi: 0.8684 | LR: 8.60e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2310s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0035 | max_backward: 0.1413 | max_optimizer_step: 0.1078 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3632.8 | uss_MB: 3142.9 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:55:28,473] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:43,264] INFO : Step 13000/50000 rank0 | Train Loss: 9.9116 | Train Loss Unweighted: 3.5848 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.1932 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1730 | Regression_eta: 0.0129 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8357 | Classification_binary: 3.1932 | Regression_cos_phi: 0.4354 | Regression_energy: 1.8913 | Regression_eta: 0.5767 | Regression_pt: 1.9745 | Regression_sin_phi: 1.0048 | LR: 8.58e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1122 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3632.8 | uss_MB: 3145.9 | child_rss_MB: 3206.5 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:55:43,476] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:43,494] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 11:55:43,684] INFO : Running validation on rank0 +rank0 [2026-10-28 11:55:43,684] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:55:43,684] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:55:44,767] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 11:55:44,911] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:55:45,042] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:55:45,116] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:55:45,184] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:55:45,251] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:55:45,320] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:55:45,389] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:55:45,496] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:55:45,626] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:55:45,756] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:55:45,826] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:55:45,894] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:55:45,962] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:55:46,031] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:55:46,099] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:55:46,203] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:55:46,209] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:55:46,212] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:55:46,214] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:55:46,215] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:55:46,218] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:55:46,221] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:55:46,222] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:55:46,224] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:55:46,226] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:55:46,228] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:55:46,230] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:55:46,231] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:55:46,234] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:55:46,235] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:55:46,237] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:55:52,088] INFO : VALIDATION | Step=13000/50000 | Train Loss=9.9116 | Valid Loss=10.0401 | Train Loss Unweighted=3.5848 | Valid Loss Unweighted=3.6257 | Stale=0 | ETA=123.3m +rank0 [2026-10-28 11:55:52,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:52,153] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:55:52,157] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:55:52,305] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:53,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:54,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:55,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:55,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:55,882] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:55:55,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:55,940] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:55:55,943] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:56,026] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:57,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:57,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:58,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:59,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:59,123] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:55:59,126] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:59,180] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:55:59,183] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:55:59,268] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:00,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:01,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:01,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:02,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:02,521] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:56:02,529] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:56:02,783] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:56:03,189] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:04,628] INFO : Plotted jets +rank0 [2026-10-28 11:56:05,709] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:05,741] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:05,970] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:06,361] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:08,187] INFO : Plotted jets +rank0 [2026-10-28 11:56:09,289] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:09,322] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 11:56:09,527] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 11:56:09,892] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:11,273] INFO : Plotted jets +rank0 [2026-10-28 11:56:12,319] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:27,259] INFO : Step 13100/50000 rank0 | Train Loss: 10.3707 | Train Loss Unweighted: 3.5507 | Train Loss Components: Classification: 0.0238 | Classification_binary: 3.1203 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1893 | Regression_eta: 0.0158 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8954 | Classification_binary: 3.1203 | Regression_cos_phi: 0.5779 | Regression_energy: 2.0694 | Regression_eta: 0.7032 | Regression_pt: 2.1869 | Regression_sin_phi: 0.8177 | LR: 8.56e-04 | DataLoad Time: 0.0198s | Model Forward Time: 1.3487s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1045 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3826.7 | uss_MB: 3338.2 | child_rss_MB: 3209.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:56:27,471] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:42,171] INFO : Step 13200/50000 rank0 | Train Loss: 11.3362 | Train Loss Unweighted: 3.6603 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.1853 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2119 | Regression_eta: 0.0237 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8331 | Classification_binary: 3.1853 | Regression_cos_phi: 0.6062 | Regression_energy: 2.3167 | Regression_eta: 1.0552 | Regression_pt: 2.3466 | Regression_sin_phi: 0.9931 | LR: 8.54e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2562s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.2871 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3827.7 | uss_MB: 3338.2 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:56:42,402] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:56,883] INFO : Step 13300/50000 rank0 | Train Loss: 10.1262 | Train Loss Unweighted: 3.5564 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.1313 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1878 | Regression_eta: 0.0132 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8179 | Classification_binary: 3.1313 | Regression_cos_phi: 0.5104 | Regression_energy: 2.0528 | Regression_eta: 0.5870 | Regression_pt: 2.2096 | Regression_sin_phi: 0.8172 | LR: 8.52e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2289s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0165 | max_loss: 0.0032 | max_backward: 0.0385 | max_optimizer_step: 0.1625 | max_tensorboard_logging: 0.2998 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3706.2 | uss_MB: 3218.1 | child_rss_MB: 3206.9 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:56:57,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:11,579] INFO : Step 13400/50000 rank0 | Train Loss: 9.4509 | Train Loss Unweighted: 3.6335 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.2492 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1719 | Regression_eta: 0.0107 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7766 | Classification_binary: 3.2492 | Regression_cos_phi: 0.4707 | Regression_energy: 1.8788 | Regression_eta: 0.4757 | Regression_pt: 1.9903 | Regression_sin_phi: 0.6097 | LR: 8.50e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2559s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0030 | max_backward: 0.0329 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3706.2 | uss_MB: 3217.1 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:57:11,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:26,275] INFO : Step 13500/50000 rank0 | Train Loss: 10.9049 | Train Loss Unweighted: 3.7137 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.2672 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1980 | Regression_eta: 0.0168 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9430 | Classification_binary: 3.2672 | Regression_cos_phi: 0.6029 | Regression_energy: 2.1651 | Regression_eta: 0.7471 | Regression_pt: 2.2252 | Regression_sin_phi: 0.9545 | LR: 8.48e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0340 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3706.2 | uss_MB: 3217.3 | child_rss_MB: 3206.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:26,502] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:41,007] INFO : Step 13600/50000 rank0 | Train Loss: 9.0932 | Train Loss Unweighted: 3.4680 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0927 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1663 | Regression_eta: 0.0100 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8342 | Classification_binary: 3.0927 | Regression_cos_phi: 0.4130 | Regression_energy: 1.8176 | Regression_eta: 0.4481 | Regression_pt: 1.9600 | Regression_sin_phi: 0.5275 | LR: 8.45e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1732s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0171 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1297 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3707.2 | uss_MB: 3220.0 | child_rss_MB: 3204.2 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:57:41,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:55,711] INFO : Step 13700/50000 rank0 | Train Loss: 9.4003 | Train Loss Unweighted: 3.4982 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.1359 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1585 | Regression_eta: 0.0121 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8471 | Classification_binary: 3.1359 | Regression_cos_phi: 0.5673 | Regression_energy: 1.7323 | Regression_eta: 0.5405 | Regression_pt: 1.8106 | Regression_sin_phi: 0.7665 | LR: 8.43e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2686s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0166 | max_loss: 0.0032 | max_backward: 0.0379 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3708.2 | uss_MB: 3220.3 | child_rss_MB: 3203.9 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 11:57:55,944] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:10,411] INFO : Step 13800/50000 rank0 | Train Loss: 9.7937 | Train Loss Unweighted: 3.6255 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.2366 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1707 | Regression_eta: 0.0122 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9230 | Classification_binary: 3.2366 | Regression_cos_phi: 0.5119 | Regression_energy: 1.8657 | Regression_eta: 0.5426 | Regression_pt: 1.9707 | Regression_sin_phi: 0.7432 | LR: 8.41e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2280s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.2159 | max_tensorboard_logging: 0.2767 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3708.2 | uss_MB: 3218.4 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:58:10,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:24,749] INFO : Step 13900/50000 rank0 | Train Loss: 10.1377 | Train Loss Unweighted: 3.6079 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.1934 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1858 | Regression_eta: 0.0142 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8808 | Classification_binary: 3.1934 | Regression_cos_phi: 0.5914 | Regression_energy: 2.0315 | Regression_eta: 0.6342 | Regression_pt: 2.0695 | Regression_sin_phi: 0.7370 | LR: 8.39e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0841 | max_loss: 0.0031 | max_backward: 0.0337 | max_optimizer_step: 0.1115 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3706.4 | uss_MB: 3216.3 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:58:24,962] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:39,510] INFO : Step 14000/50000 rank0 | Train Loss: 9.9364 | Train Loss Unweighted: 3.4252 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0045 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1892 | Regression_eta: 0.0128 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8269 | Classification_binary: 3.0045 | Regression_cos_phi: 0.4507 | Regression_energy: 2.0689 | Regression_eta: 0.5692 | Regression_pt: 2.1461 | Regression_sin_phi: 0.8701 | LR: 8.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2662s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0566 | max_forward: 0.0170 | max_loss: 0.0032 | max_backward: 0.0383 | max_optimizer_step: 0.1085 | max_tensorboard_logging: 0.3506 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3707.4 | uss_MB: 3220.0 | child_rss_MB: 3206.1 | allocated_MB: 210.7 | reserved_MB: 2770.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 11:58:39,721] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:58:39,794] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 11:58:39,877] INFO : Running validation on rank0 +rank0 [2026-10-28 11:58:39,877] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:58:39,877] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:58:40,958] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 11:58:41,104] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:58:41,236] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:58:41,310] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:58:41,379] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:58:41,446] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:58:41,516] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:58:41,585] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:58:41,691] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:58:41,813] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:58:41,935] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:58:42,008] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:58:42,077] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:58:42,145] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:58:42,214] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:58:42,283] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:58:42,391] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:58:42,398] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:58:42,401] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:58:42,404] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:58:42,406] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:58:42,407] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:58:42,410] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:58:42,411] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:58:42,414] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:58:42,416] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:58:42,418] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:58:42,421] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:58:42,424] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:58:42,427] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:58:42,430] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:58:42,433] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:58:48,245] INFO : VALIDATION | Step=14000/50000 | Train Loss=9.9364 | Valid Loss=9.9343 | Train Loss Unweighted=3.4252 | Valid Loss Unweighted=3.5593 | Stale=0 | ETA=118.9m +rank0 [2026-10-28 11:58:48,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:48,310] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:58:48,314] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:58:48,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:49,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:50,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:51,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:51,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:51,691] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:51,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:51,750] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:58:51,753] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:51,840] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:53,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:53,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:54,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:54,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:55,083] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:55,086] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:55,138] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:58:55,141] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:58:55,231] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:56,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:57,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:57,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:58,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:58,397] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:58,404] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:58:58,664] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:58:59,072] INFO : Plotted number of elements +rank0 [2026-10-28 11:59:00,520] INFO : Plotted jets +rank0 [2026-10-28 11:59:01,594] INFO : Plotted jet ratio +rank0 [2026-10-28 11:59:01,625] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:59:01,856] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:59:02,247] INFO : Plotted number of elements +rank0 [2026-10-28 11:59:04,055] INFO : Plotted jets +rank0 [2026-10-28 11:59:05,163] INFO : Plotted jet ratio +rank0 [2026-10-28 11:59:05,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 11:59:05,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 11:59:05,764] INFO : Plotted number of elements +rank0 [2026-10-28 11:59:07,146] INFO : Plotted jets +rank0 [2026-10-28 11:59:08,192] INFO : Plotted jet ratio +rank0 [2026-10-28 11:59:23,028] INFO : Step 14100/50000 rank0 | Train Loss: 9.3133 | Train Loss Unweighted: 3.4521 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.0713 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1691 | Regression_eta: 0.0102 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8477 | Classification_binary: 3.0713 | Regression_cos_phi: 0.4207 | Regression_energy: 1.8487 | Regression_eta: 0.4531 | Regression_pt: 1.9601 | Regression_sin_phi: 0.7117 | LR: 8.35e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1134 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3885 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3922.4 | uss_MB: 3434.0 | child_rss_MB: 3206.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:59:23,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:37,664] INFO : Step 14200/50000 rank0 | Train Loss: 10.1568 | Train Loss Unweighted: 3.3915 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9953 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1711 | Regression_eta: 0.0251 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8071 | Classification_binary: 2.9953 | Regression_cos_phi: 0.6147 | Regression_energy: 1.8709 | Regression_eta: 1.1188 | Regression_pt: 1.9021 | Regression_sin_phi: 0.8478 | LR: 8.32e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2841s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0330 | max_backward: 0.0329 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3803 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3922.4 | uss_MB: 3432.9 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:59:37,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:52,161] INFO : Step 14300/50000 rank0 | Train Loss: 9.1070 | Train Loss Unweighted: 3.4337 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.0592 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1669 | Regression_eta: 0.0096 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8477 | Classification_binary: 3.0592 | Regression_cos_phi: 0.4087 | Regression_energy: 1.8248 | Regression_eta: 0.4265 | Regression_pt: 1.9345 | Regression_sin_phi: 0.6055 | LR: 8.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2465s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0034 | max_backward: 0.0985 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.3079 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3779.5 | uss_MB: 3291.9 | child_rss_MB: 3227.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:59:52,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:06,735] INFO : Step 14400/50000 rank0 | Train Loss: 10.2774 | Train Loss Unweighted: 3.5045 | Train Loss Components: Classification: 0.0229 | Classification_binary: 3.0739 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1949 | Regression_eta: 0.0188 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8608 | Classification_binary: 3.0739 | Regression_cos_phi: 0.4998 | Regression_energy: 2.1304 | Regression_eta: 0.8391 | Regression_pt: 2.1201 | Regression_sin_phi: 0.7533 | LR: 8.28e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3779.5 | uss_MB: 3292.9 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:06,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:21,399] INFO : Step 14500/50000 rank0 | Train Loss: 9.7857 | Train Loss Unweighted: 3.5340 | Train Loss Components: Classification: 0.0219 | Classification_binary: 3.1233 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1812 | Regression_eta: 0.0123 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8253 | Classification_binary: 3.1233 | Regression_cos_phi: 0.4008 | Regression_energy: 1.9814 | Regression_eta: 0.5498 | Regression_pt: 2.1512 | Regression_sin_phi: 0.7540 | LR: 8.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2554s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0141 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3779.5 | uss_MB: 3293.0 | child_rss_MB: 3200.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:00:21,631] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:35,473] INFO : Step 14600/50000 rank0 | Train Loss: 9.7258 | Train Loss Unweighted: 3.5714 | Train Loss Components: Classification: 0.0218 | Classification_binary: 3.1660 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1832 | Regression_eta: 0.0133 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8206 | Classification_binary: 3.1660 | Regression_cos_phi: 0.4135 | Regression_energy: 2.0026 | Regression_eta: 0.5954 | Regression_pt: 2.0641 | Regression_sin_phi: 0.6637 | LR: 8.24e-04 | DataLoad Time: 0.0337s | Model Forward Time: 1.2034s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0035 | max_backward: 0.0763 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.2930 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3779.5 | uss_MB: 3293.0 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:35,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:49,731] INFO : Step 14700/50000 rank0 | Train Loss: 10.0717 | Train Loss Unweighted: 3.5380 | Train Loss Components: Classification: 0.0216 | Classification_binary: 3.1121 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1897 | Regression_eta: 0.0125 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8113 | Classification_binary: 3.1121 | Regression_cos_phi: 0.5481 | Regression_energy: 2.0739 | Regression_eta: 0.5564 | Regression_pt: 2.2071 | Regression_sin_phi: 0.7628 | LR: 8.21e-04 | DataLoad Time: 0.0290s | Model Forward Time: 1.2946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0561 | max_loss: 0.0034 | max_backward: 0.0399 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3798 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3779.5 | uss_MB: 3295.9 | child_rss_MB: 3208.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:00:49,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:04,051] INFO : Step 14800/50000 rank0 | Train Loss: 9.6984 | Train Loss Unweighted: 3.5725 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.1954 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1649 | Regression_eta: 0.0143 | Regression_pt: 0.1592 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8370 | Classification_binary: 3.1954 | Regression_cos_phi: 0.5023 | Regression_energy: 1.8031 | Regression_eta: 0.6381 | Regression_pt: 1.8914 | Regression_sin_phi: 0.8310 | LR: 8.19e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2661s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0169 | max_loss: 0.0033 | max_backward: 0.0383 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.2861 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3779.5 | uss_MB: 3295.9 | child_rss_MB: 3205.7 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:01:04,279] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:18,465] INFO : Step 14900/50000 rank0 | Train Loss: 10.0613 | Train Loss Unweighted: 3.5741 | Train Loss Components: Classification: 0.0226 | Classification_binary: 3.1498 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1886 | Regression_eta: 0.0115 | Regression_pt: 0.1854 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8519 | Classification_binary: 3.1498 | Regression_cos_phi: 0.5143 | Regression_energy: 2.0619 | Regression_eta: 0.5146 | Regression_pt: 2.2034 | Regression_sin_phi: 0.7653 | LR: 8.17e-04 | DataLoad Time: 0.0340s | Model Forward Time: 1.2802s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0535 | max_forward: 0.0154 | max_loss: 0.0036 | max_backward: 0.0335 | max_optimizer_step: 0.1121 | max_tensorboard_logging: 0.3629 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3778.3 | uss_MB: 3292.1 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:01:18,679] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:32,488] INFO : Step 15000/50000 rank0 | Train Loss: 9.8664 | Train Loss Unweighted: 3.3728 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.9815 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1672 | Regression_eta: 0.0190 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9226 | Classification_binary: 2.9815 | Regression_cos_phi: 0.5356 | Regression_energy: 1.8278 | Regression_eta: 0.8455 | Regression_pt: 1.9471 | Regression_sin_phi: 0.8062 | LR: 8.15e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.2739 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3778.3 | uss_MB: 3293.3 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:01:32,700] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:32,750] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 12:01:32,832] INFO : Running validation on rank0 +rank0 [2026-10-28 12:01:32,832] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:01:32,833] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:01:33,913] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 12:01:34,058] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:01:34,190] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:01:34,264] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:01:34,333] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:01:34,401] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:01:34,470] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:01:34,538] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:01:34,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:01:34,778] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:01:34,909] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:01:34,980] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:01:35,050] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:01:35,119] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:01:35,187] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:01:35,257] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:01:35,367] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:01:35,374] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:01:35,377] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:01:35,380] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:01:35,381] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:01:35,383] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:01:35,384] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:01:35,387] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:01:35,390] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:01:35,393] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:01:35,394] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:01:35,396] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:01:35,399] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:01:35,400] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:01:35,402] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:01:35,403] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:01:41,173] INFO : VALIDATION | Step=15000/50000 | Train Loss=9.8664 | Valid Loss=9.7173 | Train Loss Unweighted=3.3728 | Valid Loss Unweighted=3.5113 | Stale=0 | ETA=114.6m +rank0 [2026-10-28 12:01:41,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:41,240] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:01:41,243] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:01:41,399] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:42,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:43,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:44,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:44,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:44,739] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:01:44,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:44,797] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:01:44,800] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:44,891] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:46,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:46,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:47,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:48,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:48,140] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:48,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:48,197] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:01:48,200] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:01:48,356] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:49,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:50,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:50,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:51,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:51,417] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:51,423] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:01:51,675] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:01:52,083] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:53,524] INFO : Plotted jets +rank0 [2026-10-28 12:01:54,598] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:54,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:54,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:55,249] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:57,148] INFO : Plotted jets +rank0 [2026-10-28 12:01:58,252] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:58,284] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 12:01:58,487] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 12:01:58,850] INFO : Plotted number of elements +rank0 [2026-10-28 12:02:00,229] INFO : Plotted jets +rank0 [2026-10-28 12:02:01,274] INFO : Plotted jet ratio +rank0 [2026-10-28 12:02:15,634] INFO : Step 15100/50000 rank0 | Train Loss: 9.4367 | Train Loss Unweighted: 3.5540 | Train Loss Components: Classification: 0.0219 | Classification_binary: 3.1651 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1744 | Regression_eta: 0.0114 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8249 | Classification_binary: 3.1651 | Regression_cos_phi: 0.4123 | Regression_energy: 1.9072 | Regression_eta: 0.5085 | Regression_pt: 1.9991 | Regression_sin_phi: 0.6196 | LR: 8.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 1.2644s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.1275 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.2815 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3985.0 | uss_MB: 3501.5 | child_rss_MB: 3203.1 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:02:15,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:29,912] INFO : Step 15200/50000 rank0 | Train Loss: 9.5824 | Train Loss Unweighted: 3.4403 | Train Loss Components: Classification: 0.0233 | Classification_binary: 3.0462 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1740 | Regression_eta: 0.0130 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8775 | Classification_binary: 3.0462 | Regression_cos_phi: 0.4890 | Regression_energy: 1.9023 | Regression_eta: 0.5786 | Regression_pt: 2.0088 | Regression_sin_phi: 0.6800 | LR: 8.10e-04 | DataLoad Time: 0.1018s | Model Forward Time: 1.3048s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0015 | max_forward: 0.0246 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3985.0 | uss_MB: 3501.6 | child_rss_MB: 3205.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:02:30,127] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:44,060] INFO : Step 15300/50000 rank0 | Train Loss: 9.0698 | Train Loss Unweighted: 3.4121 | Train Loss Components: Classification: 0.0209 | Classification_binary: 3.0523 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1571 | Regression_eta: 0.0126 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7866 | Classification_binary: 3.0523 | Regression_cos_phi: 0.4914 | Regression_energy: 1.7175 | Regression_eta: 0.5602 | Regression_pt: 1.8436 | Regression_sin_phi: 0.6182 | LR: 8.08e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.1927s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3985.0 | uss_MB: 3501.6 | child_rss_MB: 3205.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:02:44,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:58,215] INFO : Step 15400/50000 rank0 | Train Loss: 10.5317 | Train Loss Unweighted: 3.6995 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.2466 | Regression_cos_phi: 0.0088 | Regression_energy: 0.2042 | Regression_eta: 0.0131 | Regression_pt: 0.1968 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8459 | Classification_binary: 3.2466 | Regression_cos_phi: 0.5710 | Regression_energy: 2.2327 | Regression_eta: 0.5838 | Regression_pt: 2.3379 | Regression_sin_phi: 0.7137 | LR: 8.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2671s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0139 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3849.0 | uss_MB: 3366.0 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:02:58,442] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:12,287] INFO : Step 15500/50000 rank0 | Train Loss: 9.0509 | Train Loss Unweighted: 3.3934 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0414 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1540 | Regression_eta: 0.0130 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8279 | Classification_binary: 3.0414 | Regression_cos_phi: 0.4716 | Regression_energy: 1.6836 | Regression_eta: 0.5809 | Regression_pt: 1.7655 | Regression_sin_phi: 0.6800 | LR: 8.03e-04 | DataLoad Time: 0.0254s | Model Forward Time: 1.2363s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.1171 | max_optimizer_step: 0.1145 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3849.0 | uss_MB: 3366.1 | child_rss_MB: 3205.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:03:12,499] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:26,208] INFO : Step 15600/50000 rank0 | Train Loss: 9.6502 | Train Loss Unweighted: 3.4492 | Train Loss Components: Classification: 0.0218 | Classification_binary: 3.0612 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1706 | Regression_eta: 0.0122 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8219 | Classification_binary: 3.0612 | Regression_cos_phi: 0.7167 | Regression_energy: 1.8651 | Regression_eta: 0.5448 | Regression_pt: 1.9622 | Regression_sin_phi: 0.6783 | LR: 8.01e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2828s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0171 | max_loss: 0.0034 | max_backward: 0.0391 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3850.0 | uss_MB: 3368.8 | child_rss_MB: 3208.0 | allocated_MB: 210.7 | reserved_MB: 2838.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:03:26,435] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:40,215] INFO : Step 15700/50000 rank0 | Train Loss: 9.9752 | Train Loss Unweighted: 3.4213 | Train Loss Components: Classification: 0.0207 | Classification_binary: 3.0207 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1793 | Regression_eta: 0.0217 | Regression_pt: 0.1626 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7800 | Classification_binary: 3.0207 | Regression_cos_phi: 0.4969 | Regression_energy: 1.9599 | Regression_eta: 0.9682 | Regression_pt: 1.9323 | Regression_sin_phi: 0.8171 | LR: 7.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2586s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3850.0 | uss_MB: 3366.8 | child_rss_MB: 3204.2 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:40,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:54,203] INFO : Step 15800/50000 rank0 | Train Loss: 9.8566 | Train Loss Unweighted: 3.5601 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.1499 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1859 | Regression_eta: 0.0125 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8302 | Classification_binary: 3.1499 | Regression_cos_phi: 0.5265 | Regression_energy: 2.0325 | Regression_eta: 0.5583 | Regression_pt: 2.0722 | Regression_sin_phi: 0.6871 | LR: 7.96e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3850.0 | uss_MB: 3366.8 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:03:54,428] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:08,179] INFO : Step 15900/50000 rank0 | Train Loss: 9.4639 | Train Loss Unweighted: 3.4243 | Train Loss Components: Classification: 0.0215 | Classification_binary: 3.0460 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1649 | Regression_eta: 0.0125 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8089 | Classification_binary: 3.0460 | Regression_cos_phi: 0.5746 | Regression_energy: 1.8025 | Regression_eta: 0.5588 | Regression_pt: 1.9338 | Regression_sin_phi: 0.7393 | LR: 7.94e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2879s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0038 | max_backward: 0.0339 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3850.0 | uss_MB: 3367.2 | child_rss_MB: 3204.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:04:08,407] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:21,950] INFO : Step 16000/50000 rank0 | Train Loss: 9.0867 | Train Loss Unweighted: 3.3120 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9470 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1604 | Regression_eta: 0.0115 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8167 | Classification_binary: 2.9470 | Regression_cos_phi: 0.5495 | Regression_energy: 1.7532 | Regression_eta: 0.5117 | Regression_pt: 1.8554 | Regression_sin_phi: 0.6533 | LR: 7.91e-04 | DataLoad Time: 0.1161s | Model Forward Time: 1.1871s | Phase Times: max_pre_step_barrier: 0.0433 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1121 | max_tensorboard_logging: 0.2842 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3850.6 | uss_MB: 3366.2 | child_rss_MB: 3208.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:04:22,162] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:22,206] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 12:04:22,289] INFO : Running validation on rank0 +rank0 [2026-10-28 12:04:22,289] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:04:22,289] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:04:23,373] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 12:04:23,519] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:04:23,651] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:04:23,725] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:04:23,794] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:04:23,861] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:04:23,931] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:04:24,002] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:04:24,109] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:04:24,241] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:04:24,373] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:04:24,444] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:04:24,513] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:04:24,582] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:04:24,652] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:04:24,722] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:04:24,832] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:04:24,839] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:04:24,842] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:04:24,844] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:04:24,845] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:04:24,848] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:04:24,852] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:04:24,854] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:04:24,855] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:04:24,858] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:04:24,859] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:04:24,861] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:04:24,864] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:04:24,865] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:04:24,867] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:04:24,868] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:04:30,699] INFO : VALIDATION | Step=16000/50000 | Train Loss=9.0867 | Valid Loss=9.6727 | Train Loss Unweighted=3.3120 | Valid Loss Unweighted=3.4797 | Stale=0 | ETA=110.4m +rank0 [2026-10-28 12:04:30,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:30,766] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:04:30,769] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:30,919] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:32,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:33,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:33,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:34,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:34,630] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:04:34,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:34,689] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:04:34,692] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:34,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:36,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:36,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:37,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:38,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:38,140] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:04:38,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:38,196] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:04:38,210] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:04:38,371] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:39,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:40,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:40,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:41,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:41,437] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:04:41,444] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:41,701] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:42,104] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:43,548] INFO : Plotted jets +rank0 [2026-10-28 12:04:44,625] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:44,657] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:44,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:45,277] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:47,133] INFO : Plotted jets +rank0 [2026-10-28 12:04:48,236] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:48,267] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:48,472] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:48,835] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:50,217] INFO : Plotted jets +rank0 [2026-10-28 12:04:51,261] INFO : Plotted jet ratio +rank0 [2026-10-28 12:05:05,455] INFO : Step 16100/50000 rank0 | Train Loss: 10.4744 | Train Loss Unweighted: 3.5254 | Train Loss Components: Classification: 0.0226 | Classification_binary: 3.0642 | Regression_cos_phi: 0.0068 | Regression_energy: 0.2056 | Regression_eta: 0.0191 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8505 | Classification_binary: 3.0642 | Regression_cos_phi: 0.4426 | Regression_energy: 2.2474 | Regression_eta: 0.8539 | Regression_pt: 2.3802 | Regression_sin_phi: 0.6356 | LR: 7.89e-04 | DataLoad Time: 0.0116s | Model Forward Time: 1.3152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1045 | max_forward: 0.0149 | max_loss: 0.0036 | max_backward: 0.0338 | max_optimizer_step: 0.1124 | max_tensorboard_logging: 0.3412 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4054.2 | uss_MB: 3570.7 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:05:05,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:19,271] INFO : Step 16200/50000 rank0 | Train Loss: 9.4108 | Train Loss Unweighted: 3.4187 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.0328 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1710 | Regression_eta: 0.0104 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8706 | Classification_binary: 3.0328 | Regression_cos_phi: 0.5153 | Regression_energy: 1.8692 | Regression_eta: 0.4645 | Regression_pt: 1.9743 | Regression_sin_phi: 0.6841 | LR: 7.86e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3032s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0612 | max_backward: 0.0343 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3689 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 4054.2 | uss_MB: 3569.5 | child_rss_MB: 3209.7 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:05:19,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:33,096] INFO : Step 16300/50000 rank0 | Train Loss: 9.4145 | Train Loss Unweighted: 3.4893 | Train Loss Components: Classification: 0.0225 | Classification_binary: 3.1185 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1627 | Regression_eta: 0.0138 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8461 | Classification_binary: 3.1185 | Regression_cos_phi: 0.5067 | Regression_energy: 1.7787 | Regression_eta: 0.6168 | Regression_pt: 1.8625 | Regression_sin_phi: 0.6852 | LR: 7.84e-04 | DataLoad Time: 0.0262s | Model Forward Time: 1.2520s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0159 | max_loss: 0.0032 | max_backward: 0.0901 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4054.2 | uss_MB: 3570.8 | child_rss_MB: 3207.2 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:05:33,310] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:46,983] INFO : Step 16400/50000 rank0 | Train Loss: 9.6569 | Train Loss Unweighted: 3.3684 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9649 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1808 | Regression_eta: 0.0150 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7963 | Classification_binary: 2.9649 | Regression_cos_phi: 0.4123 | Regression_energy: 1.9771 | Regression_eta: 0.6709 | Regression_pt: 2.0400 | Regression_sin_phi: 0.7953 | LR: 7.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2587s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.2783 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3925.7 | uss_MB: 3440.3 | child_rss_MB: 3211.0 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:05:47,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:00,994] INFO : Step 16500/50000 rank0 | Train Loss: 9.9381 | Train Loss Unweighted: 3.3857 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.9655 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1862 | Regression_eta: 0.0136 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8481 | Classification_binary: 2.9655 | Regression_cos_phi: 0.5776 | Regression_energy: 2.0353 | Regression_eta: 0.6047 | Regression_pt: 2.1522 | Regression_sin_phi: 0.7547 | LR: 7.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2145s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0035 | max_backward: 0.0872 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3925.7 | uss_MB: 3440.4 | child_rss_MB: 3206.6 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:06:01,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:15,243] INFO : Step 16600/50000 rank0 | Train Loss: 9.7441 | Train Loss Unweighted: 3.3809 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.9791 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1767 | Regression_eta: 0.0142 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8296 | Classification_binary: 2.9791 | Regression_cos_phi: 0.5405 | Regression_energy: 1.9316 | Regression_eta: 0.6334 | Regression_pt: 2.0459 | Regression_sin_phi: 0.7840 | LR: 7.77e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.3483s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1197 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.3759 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3925.7 | uss_MB: 3440.4 | child_rss_MB: 3206.9 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:06:15,454] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:29,072] INFO : Step 16700/50000 rank0 | Train Loss: 10.7443 | Train Loss Unweighted: 3.6122 | Train Loss Components: Classification: 0.0223 | Classification_binary: 3.1313 | Regression_cos_phi: 0.0069 | Regression_energy: 0.2182 | Regression_eta: 0.0140 | Regression_pt: 0.2111 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8388 | Classification_binary: 3.1313 | Regression_cos_phi: 0.4451 | Regression_energy: 2.3850 | Regression_eta: 0.6225 | Regression_pt: 2.5082 | Regression_sin_phi: 0.8135 | LR: 7.74e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3490s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.1077 | max_backward: 0.0330 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3926.7 | uss_MB: 3439.4 | child_rss_MB: 3204.7 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:06:29,285] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:42,838] INFO : Step 16800/50000 rank0 | Train Loss: 9.9643 | Train Loss Unweighted: 3.4441 | Train Loss Components: Classification: 0.0215 | Classification_binary: 3.0469 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1718 | Regression_eta: 0.0189 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8107 | Classification_binary: 3.0469 | Regression_cos_phi: 0.5943 | Regression_energy: 1.8782 | Regression_eta: 0.8451 | Regression_pt: 1.9886 | Regression_sin_phi: 0.8006 | LR: 7.72e-04 | DataLoad Time: 0.0527s | Model Forward Time: 1.2436s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0035 | max_backward: 0.1412 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2787 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3926.7 | uss_MB: 3440.5 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:06:43,051] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:56,401] INFO : Step 16900/50000 rank0 | Train Loss: 9.4625 | Train Loss Unweighted: 3.3970 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.0477 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1479 | Regression_eta: 0.0210 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8314 | Classification_binary: 3.0477 | Regression_cos_phi: 0.5292 | Regression_energy: 1.6167 | Regression_eta: 0.9351 | Regression_pt: 1.6817 | Regression_sin_phi: 0.8206 | LR: 7.69e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.3045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0642 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3927.7 | uss_MB: 3443.0 | child_rss_MB: 3203.3 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:06:56,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:10,327] INFO : Step 17000/50000 rank0 | Train Loss: 9.6699 | Train Loss Unweighted: 3.4537 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0525 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1803 | Regression_eta: 0.0163 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8362 | Classification_binary: 3.0525 | Regression_cos_phi: 0.4481 | Regression_energy: 1.9716 | Regression_eta: 0.7289 | Regression_pt: 2.0052 | Regression_sin_phi: 0.6273 | LR: 7.67e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.3225s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0898 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3926.6 | uss_MB: 3440.4 | child_rss_MB: 3208.0 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:07:10,539] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:10,656] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 12:07:10,737] INFO : Running validation on rank0 +rank0 [2026-10-28 12:07:10,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:07:10,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:07:11,831] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 12:07:11,976] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:07:12,108] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:07:12,182] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:07:12,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:07:12,319] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:07:12,389] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:07:12,458] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:07:12,565] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:07:12,697] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:07:12,828] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:07:12,899] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:07:12,969] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:07:13,038] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:07:13,108] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:07:13,178] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:07:13,290] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:07:13,297] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:07:13,300] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:07:13,302] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:07:13,304] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:07:13,306] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:07:13,309] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:07:13,310] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:07:13,312] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:07:13,315] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:07:13,316] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:07:13,319] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:07:13,322] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:07:13,323] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:07:13,325] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:07:13,328] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:07:19,399] INFO : VALIDATION | Step=17000/50000 | Train Loss=9.6699 | Valid Loss=9.5219 | Train Loss Unweighted=3.4537 | Valid Loss Unweighted=3.4338 | Stale=0 | ETA=106.3m +rank0 [2026-10-28 12:07:19,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:07:19,464] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:07:19,468] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:07:19,621] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:07:21,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:07:21,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:07:22,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:07:22,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:07:23,067] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:07:23,070] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:07:23,125] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:07:23,128] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:07:23,287] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:07:24,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:07:25,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:07:25,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:07:26,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:07:26,439] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:07:26,442] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:07:26,495] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:07:26,498] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:07:26,587] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:07:27,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:07:28,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:07:29,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:07:29,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:07:29,648] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:07:29,655] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:07:29,915] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:07:30,320] INFO : Plotted number of elements +rank0 [2026-10-28 12:07:31,753] INFO : Plotted jets +rank0 [2026-10-28 12:07:32,832] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:32,865] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:07:33,095] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:07:33,489] INFO : Plotted number of elements +rank0 [2026-10-28 12:07:35,316] INFO : Plotted jets +rank0 [2026-10-28 12:07:36,422] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:36,454] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 12:07:36,662] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 12:07:37,027] INFO : Plotted number of elements +rank0 [2026-10-28 12:07:38,403] INFO : Plotted jets +rank0 [2026-10-28 12:07:39,450] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:53,199] INFO : Step 17100/50000 rank0 | Train Loss: 9.6612 | Train Loss Unweighted: 3.4378 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0353 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1802 | Regression_eta: 0.0137 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8367 | Classification_binary: 3.0353 | Regression_cos_phi: 0.4736 | Regression_energy: 1.9703 | Regression_eta: 0.6120 | Regression_pt: 2.0402 | Regression_sin_phi: 0.6931 | LR: 7.64e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1994s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.2821 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4132.1 | uss_MB: 3648.6 | child_rss_MB: 3205.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:07:53,425] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:06,564] INFO : Step 17200/50000 rank0 | Train Loss: 9.0897 | Train Loss Unweighted: 3.3886 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.0318 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1564 | Regression_eta: 0.0126 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8878 | Classification_binary: 3.0318 | Regression_cos_phi: 0.4772 | Regression_energy: 1.7094 | Regression_eta: 0.5614 | Regression_pt: 1.7843 | Regression_sin_phi: 0.6378 | LR: 7.62e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2642s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0033 | max_backward: 0.1201 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.2763 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4132.1 | uss_MB: 3648.6 | child_rss_MB: 3206.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:08:06,776] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:20,128] INFO : Step 17300/50000 rank0 | Train Loss: 9.2359 | Train Loss Unweighted: 3.4220 | Train Loss Components: Classification: 0.0197 | Classification_binary: 3.0338 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1743 | Regression_eta: 0.0120 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7414 | Classification_binary: 3.0338 | Regression_cos_phi: 0.4133 | Regression_energy: 1.9061 | Regression_eta: 0.5353 | Regression_pt: 2.0143 | Regression_sin_phi: 0.5916 | LR: 7.59e-04 | DataLoad Time: 0.0085s | Model Forward Time: 1.2761s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0361 | max_loss: 0.0035 | max_backward: 0.0330 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4132.1 | uss_MB: 3648.7 | child_rss_MB: 3205.9 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:08:20,343] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:33,798] INFO : Step 17400/50000 rank0 | Train Loss: 9.8186 | Train Loss Unweighted: 3.4440 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.0540 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1716 | Regression_eta: 0.0153 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8425 | Classification_binary: 3.0540 | Regression_cos_phi: 0.5882 | Regression_energy: 1.8764 | Regression_eta: 0.6808 | Regression_pt: 1.9333 | Regression_sin_phi: 0.8435 | LR: 7.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2385s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0849 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3996.6 | uss_MB: 3514.7 | child_rss_MB: 3228.1 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:08:34,012] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:47,683] INFO : Step 17500/50000 rank0 | Train Loss: 9.7547 | Train Loss Unweighted: 3.4522 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0522 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1746 | Regression_eta: 0.0123 | Regression_pt: 0.1735 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8577 | Classification_binary: 3.0522 | Regression_cos_phi: 0.5933 | Regression_energy: 1.9083 | Regression_eta: 0.5473 | Regression_pt: 2.0616 | Regression_sin_phi: 0.7342 | LR: 7.54e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2825s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0170 | max_loss: 0.0034 | max_backward: 0.1442 | max_optimizer_step: 0.1131 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3996.6 | uss_MB: 3516.0 | child_rss_MB: 3203.8 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:08:47,896] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:01,256] INFO : Step 17600/50000 rank0 | Train Loss: 9.1861 | Train Loss Unweighted: 3.3536 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.9685 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1721 | Regression_eta: 0.0096 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7876 | Classification_binary: 2.9685 | Regression_cos_phi: 0.4602 | Regression_energy: 1.8820 | Regression_eta: 0.4284 | Regression_pt: 2.0009 | Regression_sin_phi: 0.6586 | LR: 7.52e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3341s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0870 | max_loss: 0.0034 | max_backward: 0.0323 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.3799 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3997.6 | uss_MB: 3514.0 | child_rss_MB: 3204.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:01,471] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:14,911] INFO : Step 17700/50000 rank0 | Train Loss: 9.0973 | Train Loss Unweighted: 3.3805 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.0201 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1595 | Regression_eta: 0.0134 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7748 | Classification_binary: 3.0201 | Regression_cos_phi: 0.4690 | Regression_energy: 1.7438 | Regression_eta: 0.5975 | Regression_pt: 1.8111 | Regression_sin_phi: 0.6810 | LR: 7.49e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.1958s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1451 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3997.6 | uss_MB: 3514.1 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:09:15,137] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:28,207] INFO : Step 17800/50000 rank0 | Train Loss: 9.0643 | Train Loss Unweighted: 3.3253 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9493 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1697 | Regression_eta: 0.0123 | Regression_pt: 0.1598 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8047 | Classification_binary: 2.9493 | Regression_cos_phi: 0.4385 | Regression_energy: 1.8556 | Regression_eta: 0.5491 | Regression_pt: 1.8987 | Regression_sin_phi: 0.5685 | LR: 7.47e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.1909s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0034 | max_backward: 0.0324 | max_optimizer_step: 0.1612 | max_tensorboard_logging: 0.2722 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3514.3 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:09:28,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:41,520] INFO : Step 17900/50000 rank0 | Train Loss: 8.8853 | Train Loss Unweighted: 3.2572 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.9051 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1571 | Regression_eta: 0.0120 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7368 | Classification_binary: 2.9051 | Regression_cos_phi: 0.4958 | Regression_energy: 1.7178 | Regression_eta: 0.5362 | Regression_pt: 1.7582 | Regression_sin_phi: 0.7354 | LR: 7.44e-04 | DataLoad Time: 0.0390s | Model Forward Time: 1.2259s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0035 | max_backward: 0.1411 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.2636 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3514.4 | child_rss_MB: 3207.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:09:41,735] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:54,779] INFO : Step 18000/50000 rank0 | Train Loss: 10.1576 | Train Loss Unweighted: 3.4926 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.0708 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1847 | Regression_eta: 0.0169 | Regression_pt: 0.1822 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8180 | Classification_binary: 3.0708 | Regression_cos_phi: 0.4766 | Regression_energy: 2.0188 | Regression_eta: 0.7559 | Regression_pt: 2.1644 | Regression_sin_phi: 0.8530 | LR: 7.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1900s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1491 | max_tensorboard_logging: 0.2857 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3997.1 | uss_MB: 3513.0 | child_rss_MB: 3208.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:55,011] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:55,029] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 12:09:55,111] INFO : Running validation on rank0 +rank0 [2026-10-28 12:09:55,112] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:09:55,112] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:09:56,312] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 12:09:56,457] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:09:56,588] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:09:56,662] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:09:56,731] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:09:56,799] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:09:56,869] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:09:56,940] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:09:57,046] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:09:57,179] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:09:57,309] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:09:57,381] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:09:57,450] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:09:57,519] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:09:57,589] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:09:57,658] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:09:57,777] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:09:57,785] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:09:57,787] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:09:57,789] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:09:57,792] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:09:57,794] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:09:57,795] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:09:57,798] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:09:57,801] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:09:57,802] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:09:57,804] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:09:57,805] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:09:57,808] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:09:57,811] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:09:57,814] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:09:57,815] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:10:03,649] INFO : VALIDATION | Step=18000/50000 | Train Loss=10.1576 | Valid Loss=9.4119 | Train Loss Unweighted=3.4926 | Valid Loss Unweighted=3.4015 | Stale=0 | ETA=102.2m +rank0 [2026-10-28 12:10:03,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:03,714] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:10:03,718] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:10:03,875] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:05,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:05,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:06,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:07,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:07,273] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:10:07,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:07,332] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:10:07,335] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:07,498] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:08,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:09,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:10,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:10,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:10,763] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:10:10,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:10,821] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:10:10,825] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:10:10,989] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:12,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:13,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:13,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:14,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:14,287] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:10:14,294] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:10:14,551] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:10:14,956] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:16,397] INFO : Plotted jets +rank0 [2026-10-28 12:10:17,476] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:17,508] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:17,735] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:18,126] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:19,985] INFO : Plotted jets +rank0 [2026-10-28 12:10:21,088] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:21,120] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 12:10:21,326] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 12:10:21,691] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:23,072] INFO : Plotted jets +rank0 [2026-10-28 12:10:24,127] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:37,883] INFO : Step 18100/50000 rank0 | Train Loss: 8.9137 | Train Loss Unweighted: 3.2944 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.9349 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1615 | Regression_eta: 0.0101 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8298 | Classification_binary: 2.9349 | Regression_cos_phi: 0.4777 | Regression_energy: 1.7661 | Regression_eta: 0.4508 | Regression_pt: 1.8009 | Regression_sin_phi: 0.6535 | LR: 7.39e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2650s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4198.7 | uss_MB: 3715.8 | child_rss_MB: 3213.8 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:10:38,111] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:51,444] INFO : Step 18200/50000 rank0 | Train Loss: 9.4016 | Train Loss Unweighted: 3.3525 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9876 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1589 | Regression_eta: 0.0131 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8038 | Classification_binary: 2.9876 | Regression_cos_phi: 0.5212 | Regression_energy: 1.7372 | Regression_eta: 0.5857 | Regression_pt: 1.8252 | Regression_sin_phi: 0.9410 | LR: 7.36e-04 | DataLoad Time: 0.0532s | Model Forward Time: 1.3925s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.1095 | max_backward: 0.0343 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.3810 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4198.7 | uss_MB: 3714.7 | child_rss_MB: 3208.7 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:10:51,657] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:05,107] INFO : Step 18300/50000 rank0 | Train Loss: 9.9069 | Train Loss Unweighted: 3.4297 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.9981 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1951 | Regression_eta: 0.0100 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7769 | Classification_binary: 2.9981 | Regression_cos_phi: 0.4927 | Regression_energy: 2.1326 | Regression_eta: 0.4475 | Regression_pt: 2.2554 | Regression_sin_phi: 0.8037 | LR: 7.34e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0173 | max_loss: 0.0034 | max_backward: 0.0382 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.2818 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4198.7 | uss_MB: 3717.9 | child_rss_MB: 3213.4 | allocated_MB: 210.7 | reserved_MB: 2752.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:11:05,334] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:18,400] INFO : Step 18400/50000 rank0 | Train Loss: 9.6908 | Train Loss Unweighted: 3.3938 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0062 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1735 | Regression_eta: 0.0176 | Regression_pt: 0.1579 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8574 | Classification_binary: 3.0062 | Regression_cos_phi: 0.4922 | Regression_energy: 1.8972 | Regression_eta: 0.7842 | Regression_pt: 1.8757 | Regression_sin_phi: 0.7779 | LR: 7.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2230s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.3013 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4071.7 | uss_MB: 3587.5 | child_rss_MB: 3214.5 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:11:18,626] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:31,691] INFO : Step 18500/50000 rank0 | Train Loss: 9.0303 | Train Loss Unweighted: 3.3196 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.9558 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1590 | Regression_eta: 0.0082 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8504 | Classification_binary: 2.9558 | Regression_cos_phi: 0.5259 | Regression_energy: 1.7388 | Regression_eta: 0.3650 | Regression_pt: 1.8808 | Regression_sin_phi: 0.7136 | LR: 7.29e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1462 | max_tensorboard_logging: 0.2725 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4071.7 | uss_MB: 3588.6 | child_rss_MB: 3217.1 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:11:31,916] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:44,991] INFO : Step 18600/50000 rank0 | Train Loss: 9.0329 | Train Loss Unweighted: 3.3503 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.9951 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1556 | Regression_eta: 0.0089 | Regression_pt: 0.1534 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7863 | Classification_binary: 2.9951 | Regression_cos_phi: 0.4973 | Regression_energy: 1.7015 | Regression_eta: 0.3975 | Regression_pt: 1.8222 | Regression_sin_phi: 0.8332 | LR: 7.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2111s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0134 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4071.7 | uss_MB: 3588.7 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:11:45,222] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:58,371] INFO : Step 18700/50000 rank0 | Train Loss: 9.3181 | Train Loss Unweighted: 3.3776 | Train Loss Components: Classification: 0.0208 | Classification_binary: 3.0012 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1656 | Regression_eta: 0.0102 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 3.0012 | Regression_cos_phi: 0.5107 | Regression_energy: 1.8103 | Regression_eta: 0.4532 | Regression_pt: 1.9401 | Regression_sin_phi: 0.8177 | LR: 7.24e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2514s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.1231 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4071.7 | uss_MB: 3588.7 | child_rss_MB: 3212.0 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:11:58,583] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:11,835] INFO : Step 18800/50000 rank0 | Train Loss: 9.2646 | Train Loss Unweighted: 3.2165 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.8458 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1624 | Regression_eta: 0.0208 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7463 | Classification_binary: 2.8458 | Regression_cos_phi: 0.4257 | Regression_energy: 1.7749 | Regression_eta: 0.9298 | Regression_pt: 1.8250 | Regression_sin_phi: 0.7172 | LR: 7.21e-04 | DataLoad Time: 0.0787s | Model Forward Time: 1.3655s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0983 | max_backward: 0.0333 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4071.7 | uss_MB: 3587.5 | child_rss_MB: 3217.2 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:12:12,050] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:25,118] INFO : Step 18900/50000 rank0 | Train Loss: 9.5529 | Train Loss Unweighted: 3.3922 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9934 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1811 | Regression_eta: 0.0125 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7993 | Classification_binary: 2.9934 | Regression_cos_phi: 0.5456 | Regression_energy: 1.9798 | Regression_eta: 0.5564 | Regression_pt: 2.0015 | Regression_sin_phi: 0.6769 | LR: 7.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2966s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0517 | max_backward: 0.0334 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3757 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4071.7 | uss_MB: 3587.5 | child_rss_MB: 3209.5 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:12:25,331] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:38,204] INFO : Step 19000/50000 rank0 | Train Loss: 8.8477 | Train Loss Unweighted: 3.3325 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.9850 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1527 | Regression_eta: 0.0114 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8100 | Classification_binary: 2.9850 | Regression_cos_phi: 0.5107 | Regression_energy: 1.6694 | Regression_eta: 0.5066 | Regression_pt: 1.7533 | Regression_sin_phi: 0.6126 | LR: 7.16e-04 | DataLoad Time: 0.0643s | Model Forward Time: 1.2384s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.1244 | max_optimizer_step: 0.1145 | max_tensorboard_logging: 0.2775 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4071.3 | uss_MB: 3587.7 | child_rss_MB: 3215.2 | allocated_MB: 207.0 | reserved_MB: 2752.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:12:38,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:38,435] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 12:12:38,516] INFO : Running validation on rank0 +rank0 [2026-10-28 12:12:38,516] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:12:38,516] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:12:39,717] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 12:12:39,862] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:12:39,994] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:12:40,068] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:12:40,137] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:12:40,205] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:12:40,275] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:12:40,346] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:12:40,455] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:12:40,586] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:12:40,717] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:12:40,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:12:40,859] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:12:40,928] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:12:40,997] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:12:41,068] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:12:41,180] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:12:41,187] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:12:41,190] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:12:41,191] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:12:41,194] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:12:41,196] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:12:41,198] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:12:41,200] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:12:41,203] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:12:41,204] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:12:41,207] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:12:41,209] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:12:41,210] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:12:41,213] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:12:41,216] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:12:41,217] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:12:47,127] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.8477 | Valid Loss=9.3071 | Train Loss Unweighted=3.3325 | Valid Loss Unweighted=3.3549 | Stale=0 | ETA=98.3m +rank0 [2026-10-28 12:12:47,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:47,194] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:12:47,197] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:47,357] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:48,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:49,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:50,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:50,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:50,745] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:12:50,748] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:50,804] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:12:50,807] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:50,897] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:52,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:52,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:53,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:54,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:54,239] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:12:54,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:54,296] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:12:54,299] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:12:54,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:55,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:56,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:56,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:57,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:57,527] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:12:57,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:57,796] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:58,201] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:59,641] INFO : Plotted jets +rank0 [2026-10-28 12:13:00,726] INFO : Plotted jet ratio +rank0 [2026-10-28 12:13:00,760] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:13:00,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:13:01,383] INFO : Plotted number of elements +rank0 [2026-10-28 12:13:03,220] INFO : Plotted jets +rank0 [2026-10-28 12:13:04,325] INFO : Plotted jet ratio +rank0 [2026-10-28 12:13:04,358] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 12:13:04,564] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 12:13:04,930] INFO : Plotted number of elements +rank0 [2026-10-28 12:13:06,305] INFO : Plotted jets +rank0 [2026-10-28 12:13:07,354] INFO : Plotted jet ratio +rank0 [2026-10-28 12:13:20,893] INFO : Step 19100/50000 rank0 | Train Loss: 9.3367 | Train Loss Unweighted: 3.3824 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0153 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1635 | Regression_eta: 0.0128 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8284 | Classification_binary: 3.0153 | Regression_cos_phi: 0.6239 | Regression_energy: 1.7871 | Regression_eta: 0.5692 | Regression_pt: 1.8038 | Regression_sin_phi: 0.7091 | LR: 7.13e-04 | DataLoad Time: 0.0765s | Model Forward Time: 1.2775s | Phase Times: max_pre_step_barrier: 0.0790 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4271.4 | uss_MB: 3787.2 | child_rss_MB: 3214.4 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:13:21,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:33,612] INFO : Step 19200/50000 rank0 | Train Loss: 8.8605 | Train Loss Unweighted: 3.3094 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.9431 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1633 | Regression_eta: 0.0104 | Regression_pt: 0.1599 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7335 | Classification_binary: 2.9431 | Regression_cos_phi: 0.4580 | Regression_energy: 1.7855 | Regression_eta: 0.4647 | Regression_pt: 1.9002 | Regression_sin_phi: 0.5754 | LR: 7.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2039s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0162 | max_loss: 0.0036 | max_backward: 0.0378 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4271.4 | uss_MB: 3789.3 | child_rss_MB: 3205.9 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:13:33,842] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:47,027] INFO : Step 19300/50000 rank0 | Train Loss: 9.1006 | Train Loss Unweighted: 3.2988 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.9370 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1595 | Regression_eta: 0.0122 | Regression_pt: 0.1539 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7631 | Classification_binary: 2.9370 | Regression_cos_phi: 0.5170 | Regression_energy: 1.7434 | Regression_eta: 0.5420 | Regression_pt: 1.8286 | Regression_sin_phi: 0.7695 | LR: 7.08e-04 | DataLoad Time: 0.0596s | Model Forward Time: 1.6940s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0444 | max_forward: 0.0175 | max_loss: 0.0034 | max_backward: 0.0387 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0802 | Memory: rss_MB: 4271.4 | uss_MB: 3789.4 | child_rss_MB: 3209.9 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:13:47,240] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 108.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:59,899] INFO : Step 19400/50000 rank0 | Train Loss: 9.6131 | Train Loss Unweighted: 3.3806 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.9920 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1710 | Regression_eta: 0.0129 | Regression_pt: 0.1641 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9003 | Classification_binary: 2.9920 | Regression_cos_phi: 0.5591 | Regression_energy: 1.8698 | Regression_eta: 0.5736 | Regression_pt: 1.9493 | Regression_sin_phi: 0.7689 | LR: 7.05e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3022s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0372 | max_forward: 0.0147 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3388 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4271.4 | uss_MB: 3787.3 | child_rss_MB: 3212.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:00,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:12,683] INFO : Step 19500/50000 rank0 | Train Loss: 8.8270 | Train Loss Unweighted: 3.2184 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8636 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1559 | Regression_eta: 0.0116 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7843 | Classification_binary: 2.8636 | Regression_cos_phi: 0.5091 | Regression_energy: 1.7041 | Regression_eta: 0.5176 | Regression_pt: 1.8040 | Regression_sin_phi: 0.6443 | LR: 7.02e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2183s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1424 | max_tensorboard_logging: 0.2952 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4148.1 | uss_MB: 3663.9 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:12,916] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:25,688] INFO : Step 19600/50000 rank0 | Train Loss: 9.1729 | Train Loss Unweighted: 3.2802 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9176 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1573 | Regression_eta: 0.0165 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7995 | Classification_binary: 2.9176 | Regression_cos_phi: 0.4890 | Regression_energy: 1.7199 | Regression_eta: 0.7362 | Regression_pt: 1.8131 | Regression_sin_phi: 0.6976 | LR: 7.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3488s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.3836 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4148.1 | uss_MB: 3664.0 | child_rss_MB: 3213.1 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:25,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:38,908] INFO : Step 19700/50000 rank0 | Train Loss: 9.5319 | Train Loss Unweighted: 3.3654 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.9858 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1680 | Regression_eta: 0.0183 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7715 | Classification_binary: 2.9858 | Regression_cos_phi: 0.4458 | Regression_energy: 1.8363 | Regression_eta: 0.8151 | Regression_pt: 1.8716 | Regression_sin_phi: 0.8059 | LR: 6.97e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3120s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0479 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4148.1 | uss_MB: 3664.1 | child_rss_MB: 3209.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:14:39,123] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:52,230] INFO : Step 19800/50000 rank0 | Train Loss: 9.9787 | Train Loss Unweighted: 3.3479 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.9566 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1642 | Regression_eta: 0.0248 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8351 | Classification_binary: 2.9566 | Regression_cos_phi: 0.5304 | Regression_energy: 1.7956 | Regression_eta: 1.1063 | Regression_pt: 1.9409 | Regression_sin_phi: 0.8137 | LR: 6.94e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0402 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3542 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4148.1 | uss_MB: 3664.1 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:14:52,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:05,152] INFO : Step 19900/50000 rank0 | Train Loss: 9.2461 | Train Loss Unweighted: 3.2684 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8965 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1624 | Regression_eta: 0.0160 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7676 | Classification_binary: 2.8965 | Regression_cos_phi: 0.4873 | Regression_energy: 1.7751 | Regression_eta: 0.7132 | Regression_pt: 1.8771 | Regression_sin_phi: 0.7292 | LR: 6.92e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2819s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.1317 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.2922 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4148.1 | uss_MB: 3664.2 | child_rss_MB: 3228.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:15:05,367] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:18,066] INFO : Step 20000/50000 rank0 | Train Loss: 10.4556 | Train Loss Unweighted: 3.5075 | Train Loss Components: Classification: 0.0216 | Classification_binary: 3.0704 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1958 | Regression_eta: 0.0202 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8136 | Classification_binary: 3.0704 | Regression_cos_phi: 0.5392 | Regression_energy: 2.1411 | Regression_eta: 0.9003 | Regression_pt: 2.1680 | Regression_sin_phi: 0.8230 | LR: 6.89e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3068s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0600 | max_forward: 0.0137 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.3531 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4148.1 | uss_MB: 3664.3 | child_rss_MB: 3203.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:15:18,278] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:18,314] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 12:15:18,396] INFO : Running validation on rank0 +rank0 [2026-10-28 12:15:18,397] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:15:18,397] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:15:19,503] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 12:15:19,647] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:15:19,781] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:15:19,854] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:15:19,926] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:15:19,994] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:15:20,064] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:15:20,134] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:15:20,242] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:15:20,376] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:15:20,507] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:15:20,580] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:15:20,649] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:15:20,718] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:15:20,789] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:15:20,859] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:15:20,975] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:15:20,982] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:15:20,984] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:15:20,988] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:15:20,990] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:15:20,991] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:15:20,994] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:15:20,995] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:15:20,998] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:15:21,001] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:15:21,003] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:15:21,006] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:15:21,009] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:15:21,010] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:15:21,013] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:15:21,016] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:15:27,088] INFO : VALIDATION | Step=20000/50000 | Train Loss=10.4556 | Valid Loss=9.2107 | Train Loss Unweighted=3.5075 | Valid Loss Unweighted=3.3124 | Stale=0 | ETA=94.3m +rank0 [2026-10-28 12:15:27,098] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:15:27,154] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:15:27,157] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:15:27,316] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:15:28,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:15:29,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:15:30,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:15:30,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:15:30,782] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:15:30,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:15:30,840] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:15:30,854] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:15:31,009] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:15:32,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:15:33,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:15:33,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:15:34,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:15:34,303] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:15:34,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:15:34,360] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:15:34,363] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:15:34,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:15:35,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:15:36,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:15:37,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:15:37,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:15:37,784] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:15:37,792] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:15:38,050] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:15:38,456] INFO : Plotted number of elements +rank0 [2026-10-28 12:15:39,901] INFO : Plotted jets +rank0 [2026-10-28 12:15:40,991] INFO : Plotted jet ratio +rank0 [2026-10-28 12:15:41,027] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:15:41,257] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:15:41,648] INFO : Plotted number of elements +rank0 [2026-10-28 12:15:43,503] INFO : Plotted jets +rank0 [2026-10-28 12:15:44,611] INFO : Plotted jet ratio +rank0 [2026-10-28 12:15:44,646] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 12:15:44,853] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 12:15:45,217] INFO : Plotted number of elements +rank0 [2026-10-28 12:15:46,602] INFO : Plotted jets +rank0 [2026-10-28 12:15:47,655] INFO : Plotted jet ratio +rank0 [2026-10-28 12:16:00,740] INFO : Step 20100/50000 rank0 | Train Loss: 9.7935 | Train Loss Unweighted: 3.3790 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.9950 | Regression_cos_phi: 0.0118 | Regression_energy: 0.1678 | Regression_eta: 0.0198 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7336 | Classification_binary: 2.9950 | Regression_cos_phi: 0.7653 | Regression_energy: 1.8348 | Regression_eta: 0.8833 | Regression_pt: 1.8715 | Regression_sin_phi: 0.7099 | LR: 6.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.3111 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4343.6 | uss_MB: 3860.9 | child_rss_MB: 3206.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:16:00,955] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:14,055] INFO : Step 20200/50000 rank0 | Train Loss: 9.1153 | Train Loss Unweighted: 3.2642 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8917 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1651 | Regression_eta: 0.0123 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7675 | Classification_binary: 2.8917 | Regression_cos_phi: 0.5092 | Regression_energy: 1.8052 | Regression_eta: 0.5470 | Regression_pt: 1.8947 | Regression_sin_phi: 0.6999 | LR: 6.84e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.3016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.1403 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4344.6 | uss_MB: 3861.0 | child_rss_MB: 3205.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:16:14,267] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:26,925] INFO : Step 20300/50000 rank0 | Train Loss: 9.5089 | Train Loss Unweighted: 3.4058 | Train Loss Components: Classification: 0.0230 | Classification_binary: 3.0174 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1721 | Regression_eta: 0.0124 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8664 | Classification_binary: 3.0174 | Regression_cos_phi: 0.5277 | Regression_energy: 1.8814 | Regression_eta: 0.5551 | Regression_pt: 1.9639 | Regression_sin_phi: 0.6969 | LR: 6.81e-04 | DataLoad Time: 0.0248s | Model Forward Time: 1.3971s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0801 | max_backward: 0.0331 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.4031 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4344.6 | uss_MB: 3860.9 | child_rss_MB: 3204.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:16:27,139] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:39,864] INFO : Step 20400/50000 rank0 | Train Loss: 9.5278 | Train Loss Unweighted: 3.3950 | Train Loss Components: Classification: 0.0199 | Classification_binary: 3.0173 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1664 | Regression_eta: 0.0121 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7495 | Classification_binary: 3.0173 | Regression_cos_phi: 0.5085 | Regression_energy: 1.8192 | Regression_eta: 0.5388 | Regression_pt: 1.9146 | Regression_sin_phi: 0.9799 | LR: 6.78e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3059s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0177 | max_loss: 0.0034 | max_backward: 0.0391 | max_optimizer_step: 0.1857 | max_tensorboard_logging: 0.2914 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4344.6 | uss_MB: 3863.1 | child_rss_MB: 3207.1 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:16:40,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:52,719] INFO : Step 20500/50000 rank0 | Train Loss: 9.2587 | Train Loss Unweighted: 3.3052 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9247 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1675 | Regression_eta: 0.0137 | Regression_pt: 0.1634 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8184 | Classification_binary: 2.9247 | Regression_cos_phi: 0.4553 | Regression_energy: 1.8317 | Regression_eta: 0.6095 | Regression_pt: 1.9420 | Regression_sin_phi: 0.6772 | LR: 6.76e-04 | DataLoad Time: 0.0040s | Model Forward Time: 1.3571s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0852 | max_backward: 0.0332 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4221.8 | uss_MB: 3735.5 | child_rss_MB: 3207.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:16:52,931] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:05,435] INFO : Step 20600/50000 rank0 | Train Loss: 8.3952 | Train Loss Unweighted: 3.2361 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.8944 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1512 | Regression_eta: 0.0076 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7867 | Classification_binary: 2.8944 | Regression_cos_phi: 0.3735 | Regression_energy: 1.6534 | Regression_eta: 0.3394 | Regression_pt: 1.7847 | Regression_sin_phi: 0.5632 | LR: 6.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0461 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4221.8 | uss_MB: 3736.8 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:17:05,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:18,438] INFO : Step 20700/50000 rank0 | Train Loss: 9.4802 | Train Loss Unweighted: 3.3733 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9964 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1660 | Regression_eta: 0.0142 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8445 | Classification_binary: 2.9964 | Regression_cos_phi: 0.4954 | Regression_energy: 1.8145 | Regression_eta: 0.6320 | Regression_pt: 1.8780 | Regression_sin_phi: 0.8194 | LR: 6.70e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2983s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.1410 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4221.8 | uss_MB: 3736.6 | child_rss_MB: 3207.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:17:18,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 113.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:31,061] INFO : Step 20800/50000 rank0 | Train Loss: 9.2801 | Train Loss Unweighted: 3.3489 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.9657 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1721 | Regression_eta: 0.0125 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7691 | Classification_binary: 2.9657 | Regression_cos_phi: 0.4627 | Regression_energy: 1.8812 | Regression_eta: 0.5562 | Regression_pt: 1.9449 | Regression_sin_phi: 0.7003 | LR: 6.67e-04 | DataLoad Time: 0.0575s | Model Forward Time: 1.2498s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0188 | max_loss: 0.0033 | max_backward: 0.0865 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.2864 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4221.8 | uss_MB: 3739.0 | child_rss_MB: 3230.2 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:17:31,275] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:43,668] INFO : Step 20900/50000 rank0 | Train Loss: 9.2390 | Train Loss Unweighted: 3.3702 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9963 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1663 | Regression_eta: 0.0112 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8181 | Classification_binary: 2.9963 | Regression_cos_phi: 0.4992 | Regression_energy: 1.8177 | Regression_eta: 0.5000 | Regression_pt: 1.8947 | Regression_sin_phi: 0.7131 | LR: 6.65e-04 | DataLoad Time: 0.0322s | Model Forward Time: 1.2085s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.0653 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.2939 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4220.8 | uss_MB: 3736.8 | child_rss_MB: 3229.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:17:43,880] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:56,301] INFO : Step 21000/50000 rank0 | Train Loss: 10.1144 | Train Loss Unweighted: 3.3631 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.9509 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1832 | Regression_eta: 0.0214 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7221 | Classification_binary: 2.9509 | Regression_cos_phi: 0.5213 | Regression_energy: 2.0025 | Regression_eta: 0.9554 | Regression_pt: 2.0268 | Regression_sin_phi: 0.9355 | LR: 6.62e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3371s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.1399 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3028 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 4220.8 | uss_MB: 3739.0 | child_rss_MB: 3228.2 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:17:56,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:56,536] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 12:17:56,616] INFO : Running validation on rank0 +rank0 [2026-10-28 12:17:56,617] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:17:56,617] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:17:57,808] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 12:17:57,954] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:17:58,086] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:17:58,161] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:17:58,230] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:17:58,300] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:17:58,370] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:17:58,440] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:17:58,548] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:17:58,671] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:17:58,792] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:17:58,863] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:17:58,933] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:17:59,002] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:17:59,071] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:17:59,141] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:17:59,257] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:17:59,265] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:17:59,268] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:17:59,271] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:17:59,272] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:17:59,274] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:17:59,277] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:17:59,279] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:17:59,281] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:17:59,284] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:17:59,286] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:17:59,287] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:17:59,290] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:17:59,293] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:17:59,294] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:17:59,296] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:18:05,229] INFO : VALIDATION | Step=21000/50000 | Train Loss=10.1144 | Valid Loss=9.2441 | Train Loss Unweighted=3.3631 | Valid Loss Unweighted=3.3157 | Stale=1 | ETA=90.5m +rank0 [2026-10-28 12:18:05,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:18:05,319] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:18:05,323] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:18:05,552] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:18:07,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:18:07,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:18:08,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:18:09,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:18:09,351] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:18:09,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:18:09,409] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:18:09,412] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:18:09,575] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:18:10,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:18:11,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:18:12,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:18:12,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:18:12,833] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:18:12,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:18:12,890] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:18:12,906] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:18:13,069] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:18:14,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:18:15,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:18:15,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:18:16,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:18:16,353] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:18:16,361] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:18:16,619] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:18:17,026] INFO : Plotted number of elements +rank0 [2026-10-28 12:18:18,471] INFO : Plotted jets +rank0 [2026-10-28 12:18:19,552] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:19,586] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:18:19,818] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:18:20,210] INFO : Plotted number of elements +rank0 [2026-10-28 12:18:22,100] INFO : Plotted jets +rank0 [2026-10-28 12:18:23,206] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:23,237] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 12:18:23,446] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 12:18:23,808] INFO : Plotted number of elements +rank0 [2026-10-28 12:18:25,191] INFO : Plotted jets +rank0 [2026-10-28 12:18:26,245] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:39,803] INFO : Step 21100/50000 rank0 | Train Loss: 9.5140 | Train Loss Unweighted: 3.3751 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.0026 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1626 | Regression_eta: 0.0135 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8000 | Classification_binary: 3.0026 | Regression_cos_phi: 0.6255 | Regression_energy: 1.7777 | Regression_eta: 0.6002 | Regression_pt: 1.8597 | Regression_sin_phi: 0.8482 | LR: 6.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0329 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4427.0 | uss_MB: 3945.3 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:18:40,033] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:41,296] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 12:18:41,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 12:18:51,233] INFO : Step 21200/50000 rank0 | Train Loss: 9.3283 | Train Loss Unweighted: 3.3616 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9888 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1660 | Regression_eta: 0.0146 | Regression_pt: 0.1560 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7970 | Classification_binary: 2.9888 | Regression_cos_phi: 0.4500 | Regression_energy: 1.8143 | Regression_eta: 0.6507 | Regression_pt: 1.8535 | Regression_sin_phi: 0.7739 | LR: 6.56e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3993s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0859 | max_backward: 0.0332 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3961 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4465.8 | uss_MB: 3983.2 | child_rss_MB: 3204.4 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:51,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:02,331] INFO : Step 21300/50000 rank0 | Train Loss: 8.6595 | Train Loss Unweighted: 3.2056 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.8543 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1554 | Regression_eta: 0.0103 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7761 | Classification_binary: 2.8543 | Regression_cos_phi: 0.4499 | Regression_energy: 1.6992 | Regression_eta: 0.4599 | Regression_pt: 1.7997 | Regression_sin_phi: 0.6204 | LR: 6.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3891s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0641 | max_backward: 0.0328 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.4129 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4465.8 | uss_MB: 3983.3 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:02,547] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:13,404] INFO : Step 21400/50000 rank0 | Train Loss: 8.4317 | Train Loss Unweighted: 3.1557 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.8291 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1430 | Regression_eta: 0.0098 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8022 | Classification_binary: 2.8291 | Regression_cos_phi: 0.4838 | Regression_energy: 1.5634 | Regression_eta: 0.4369 | Regression_pt: 1.6380 | Regression_sin_phi: 0.6784 | LR: 6.51e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.3229s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0220 | max_backward: 0.0333 | max_optimizer_step: 0.1131 | max_tensorboard_logging: 0.4030 | max_tensorboard_flush: 0.0131 | Memory: rss_MB: 4465.8 | uss_MB: 3983.3 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:13,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:24,484] INFO : Step 21500/50000 rank0 | Train Loss: 8.8374 | Train Loss Unweighted: 3.3401 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.9997 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1460 | Regression_eta: 0.0152 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8337 | Classification_binary: 2.9997 | Regression_cos_phi: 0.4296 | Regression_energy: 1.5961 | Regression_eta: 0.6764 | Regression_pt: 1.7137 | Regression_sin_phi: 0.5881 | LR: 6.48e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3635s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0661 | max_backward: 0.0331 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3980 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4291.4 | uss_MB: 3808.1 | child_rss_MB: 3199.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:19:24,697] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:35,619] INFO : Step 21600/50000 rank0 | Train Loss: 8.9794 | Train Loss Unweighted: 3.2363 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8504 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1734 | Regression_eta: 0.0092 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7697 | Classification_binary: 2.8504 | Regression_cos_phi: 0.4679 | Regression_energy: 1.8953 | Regression_eta: 0.4099 | Regression_pt: 2.0155 | Regression_sin_phi: 0.5707 | LR: 6.45e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3617s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0736 | max_backward: 0.0338 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3853 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4292.4 | uss_MB: 3807.6 | child_rss_MB: 3202.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:19:35,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:46,843] INFO : Step 21700/50000 rank0 | Train Loss: 8.7087 | Train Loss Unweighted: 3.2836 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.9215 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1626 | Regression_eta: 0.0109 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6924 | Classification_binary: 2.9215 | Regression_cos_phi: 0.3799 | Regression_energy: 1.7780 | Regression_eta: 0.4864 | Regression_pt: 1.8811 | Regression_sin_phi: 0.5694 | LR: 6.43e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3857s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0980 | max_backward: 0.0328 | max_optimizer_step: 0.1168 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4292.4 | uss_MB: 3808.8 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:19:47,056] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:58,096] INFO : Step 21800/50000 rank0 | Train Loss: 9.3929 | Train Loss Unweighted: 3.3557 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.9735 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1676 | Regression_eta: 0.0145 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8341 | Classification_binary: 2.9735 | Regression_cos_phi: 0.5217 | Regression_energy: 1.8322 | Regression_eta: 0.6463 | Regression_pt: 1.9389 | Regression_sin_phi: 0.6463 | LR: 6.40e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1202 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3921 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4292.4 | uss_MB: 3808.9 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:58,310] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:09,173] INFO : Step 21900/50000 rank0 | Train Loss: 9.7482 | Train Loss Unweighted: 3.3253 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.9334 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1709 | Regression_eta: 0.0224 | Regression_pt: 0.1621 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 2.9334 | Regression_cos_phi: 0.5531 | Regression_energy: 1.8686 | Regression_eta: 0.9979 | Regression_pt: 1.9255 | Regression_sin_phi: 0.6849 | LR: 6.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3250s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0697 | max_loss: 0.0031 | max_backward: 0.0331 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3948 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 4292.4 | uss_MB: 3809.0 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:20:09,385] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:20,234] INFO : Step 22000/50000 rank0 | Train Loss: 9.3696 | Train Loss Unweighted: 3.2712 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8855 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1700 | Regression_eta: 0.0132 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7633 | Classification_binary: 2.8855 | Regression_cos_phi: 0.5962 | Regression_energy: 1.8586 | Regression_eta: 0.5904 | Regression_pt: 1.9651 | Regression_sin_phi: 0.7104 | LR: 6.34e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.3109s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0573 | max_loss: 0.0029 | max_backward: 0.0328 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4292.4 | uss_MB: 3809.0 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:20:20,446] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:20,554] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 12:20:20,637] INFO : Running validation on rank0 +rank0 [2026-10-28 12:20:20,637] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:20:20,637] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:20:21,749] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 12:20:21,893] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:20:22,026] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:20:22,101] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:20:22,170] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:20:22,239] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:20:22,308] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:20:22,378] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:20:22,486] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:20:22,620] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:20:22,751] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:20:22,822] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:20:22,892] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:20:22,960] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:20:23,030] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:20:23,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:20:23,216] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:20:23,230] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:20:23,340] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:20:23,344] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:20:23,346] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:20:23,349] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:20:23,350] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:20:23,353] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:20:23,354] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:20:23,356] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:20:23,359] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:20:23,362] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:20:23,363] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:20:23,366] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:20:23,368] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:20:23,370] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:20:29,341] INFO : VALIDATION | Step=22000/50000 | Train Loss=9.3696 | Valid Loss=9.0184 | Train Loss Unweighted=3.2712 | Valid Loss Unweighted=3.2459 | Stale=0 | ETA=86.5m +rank0 [2026-10-28 12:20:29,350] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:20:29,408] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:20:29,412] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:20:29,576] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:20:31,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:20:31,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:20:32,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:20:32,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:20:32,994] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:20:32,998] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:20:33,056] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:20:33,060] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:20:33,224] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:20:34,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:20:35,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:20:35,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:20:36,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:20:36,494] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:20:36,498] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:20:36,553] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:20:36,556] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:20:36,725] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:20:38,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:20:38,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:20:39,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:20:39,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:20:39,848] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:20:39,855] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:20:40,115] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:20:40,522] INFO : Plotted number of elements +rank0 [2026-10-28 12:20:41,970] INFO : Plotted jets +rank0 [2026-10-28 12:20:43,051] INFO : Plotted jet ratio +rank0 [2026-10-28 12:20:43,084] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:20:43,314] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:20:43,710] INFO : Plotted number of elements +rank0 [2026-10-28 12:20:45,563] INFO : Plotted jets +rank0 [2026-10-28 12:20:46,671] INFO : Plotted jet ratio +rank0 [2026-10-28 12:20:46,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 12:20:46,911] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 12:20:47,275] INFO : Plotted number of elements +rank0 [2026-10-28 12:20:48,657] INFO : Plotted jets +rank0 [2026-10-28 12:20:49,710] INFO : Plotted jet ratio +rank0 [2026-10-28 12:21:00,922] INFO : Step 22100/50000 rank0 | Train Loss: 8.9134 | Train Loss Unweighted: 3.2596 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8998 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1598 | Regression_eta: 0.0095 | Regression_pt: 0.1556 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7214 | Classification_binary: 2.8998 | Regression_cos_phi: 0.4847 | Regression_energy: 1.7473 | Regression_eta: 0.4252 | Regression_pt: 1.8485 | Regression_sin_phi: 0.7866 | LR: 6.32e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0676 | max_backward: 0.0339 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.4074 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4498.8 | uss_MB: 4016.8 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:21:01,136] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:12,119] INFO : Step 22200/50000 rank0 | Train Loss: 9.3864 | Train Loss Unweighted: 3.3148 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.9348 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1678 | Regression_eta: 0.0122 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8401 | Classification_binary: 2.9348 | Regression_cos_phi: 0.4851 | Regression_energy: 1.8345 | Regression_eta: 0.5452 | Regression_pt: 1.9186 | Regression_sin_phi: 0.8281 | LR: 6.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3763s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0864 | max_loss: 0.0030 | max_backward: 0.0341 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4498.8 | uss_MB: 4016.8 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:21:12,330] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:23,179] INFO : Step 22300/50000 rank0 | Train Loss: 9.4579 | Train Loss Unweighted: 3.3217 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.9623 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1590 | Regression_eta: 0.0161 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8285 | Classification_binary: 2.9623 | Regression_cos_phi: 0.5194 | Regression_energy: 1.7380 | Regression_eta: 0.7175 | Regression_pt: 1.7115 | Regression_sin_phi: 0.9807 | LR: 6.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4367s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0809 | max_backward: 0.0330 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.4043 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4498.8 | uss_MB: 4016.8 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:21:23,391] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:34,340] INFO : Step 22400/50000 rank0 | Train Loss: 9.1994 | Train Loss Unweighted: 3.2721 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.9160 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1549 | Regression_eta: 0.0158 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9057 | Classification_binary: 2.9160 | Regression_cos_phi: 0.5012 | Regression_energy: 1.6931 | Regression_eta: 0.7027 | Regression_pt: 1.7327 | Regression_sin_phi: 0.7480 | LR: 6.23e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3340s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0942 | max_forward: 0.0170 | max_loss: 0.0033 | max_backward: 0.0386 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3262 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4498.8 | uss_MB: 4019.0 | child_rss_MB: 3203.3 | allocated_MB: 210.7 | reserved_MB: 2838.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:21:34,551] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:45,550] INFO : Step 22500/50000 rank0 | Train Loss: 8.9462 | Train Loss Unweighted: 3.2222 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8741 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1500 | Regression_eta: 0.0161 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7491 | Classification_binary: 2.8741 | Regression_cos_phi: 0.5615 | Regression_energy: 1.6395 | Regression_eta: 0.7162 | Regression_pt: 1.7405 | Regression_sin_phi: 0.6652 | LR: 6.20e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3929s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0469 | max_forward: 0.0155 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 4366.8 | uss_MB: 3883.9 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:21:45,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:56,558] INFO : Step 22600/50000 rank0 | Train Loss: 8.8771 | Train Loss Unweighted: 3.2326 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.8879 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1506 | Regression_eta: 0.0140 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7522 | Classification_binary: 2.8879 | Regression_cos_phi: 0.5052 | Regression_energy: 1.6469 | Regression_eta: 0.6252 | Regression_pt: 1.7166 | Regression_sin_phi: 0.7432 | LR: 6.18e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3467s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1032 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3421 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4366.8 | uss_MB: 3884.8 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:21:56,770] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:07,503] INFO : Step 22700/50000 rank0 | Train Loss: 9.2684 | Train Loss Unweighted: 3.2245 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.8495 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1645 | Regression_eta: 0.0120 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7862 | Classification_binary: 2.8495 | Regression_cos_phi: 0.5305 | Regression_energy: 1.7983 | Regression_eta: 0.5372 | Regression_pt: 1.9054 | Regression_sin_phi: 0.8613 | LR: 6.15e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3713s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0793 | max_backward: 0.0329 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3957 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4366.8 | uss_MB: 3884.8 | child_rss_MB: 3201.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:22:07,715] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:18,678] INFO : Step 22800/50000 rank0 | Train Loss: 9.0219 | Train Loss Unweighted: 3.3442 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.9783 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1626 | Regression_eta: 0.0106 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7243 | Classification_binary: 2.9783 | Regression_cos_phi: 0.4605 | Regression_energy: 1.7777 | Regression_eta: 0.4736 | Regression_pt: 1.8867 | Regression_sin_phi: 0.7208 | LR: 6.12e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1198 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.3919 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 4366.8 | uss_MB: 3885.0 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:22:18,895] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:22:29,876] INFO : Step 22900/50000 rank0 | Train Loss: 9.9238 | Train Loss Unweighted: 3.3371 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9352 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1763 | Regression_eta: 0.0165 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8435 | Classification_binary: 2.9352 | Regression_cos_phi: 0.7334 | Regression_energy: 1.9276 | Regression_eta: 0.7340 | Regression_pt: 1.9886 | Regression_sin_phi: 0.7615 | LR: 6.09e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3849s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0929 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.3458 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4366.8 | uss_MB: 3885.1 | child_rss_MB: 3199.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:22:30,089] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:40,890] INFO : Step 23000/50000 rank0 | Train Loss: 8.3613 | Train Loss Unweighted: 3.1511 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.8175 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1479 | Regression_eta: 0.0095 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7415 | Classification_binary: 2.8175 | Regression_cos_phi: 0.3965 | Regression_energy: 1.6172 | Regression_eta: 0.4238 | Regression_pt: 1.7043 | Regression_sin_phi: 0.6606 | LR: 6.06e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0458 | max_loss: 0.0033 | max_backward: 0.0382 | max_optimizer_step: 0.1154 | max_tensorboard_logging: 0.3887 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4366.8 | uss_MB: 3887.6 | child_rss_MB: 3204.5 | allocated_MB: 210.7 | reserved_MB: 2838.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:22:41,101] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:41,182] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 12:22:41,264] INFO : Running validation on rank0 +rank0 [2026-10-28 12:22:41,264] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:22:41,264] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:22:42,369] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 12:22:42,514] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:22:42,647] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:22:42,722] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:22:42,792] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:22:42,859] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:22:42,929] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:22:42,999] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:22:43,108] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:22:43,242] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:22:43,373] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:22:43,444] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:22:43,515] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:22:43,584] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:22:43,654] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:22:43,724] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:22:43,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:22:43,851] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:22:43,853] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:22:43,856] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:22:43,857] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:22:43,860] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:22:43,863] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:22:43,864] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:22:43,866] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:22:43,870] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:22:43,871] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:22:43,873] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:22:43,876] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:22:43,877] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:22:43,879] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:22:43,882] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:22:49,956] INFO : VALIDATION | Step=23000/50000 | Train Loss=8.3613 | Valid Loss=8.9301 | Train Loss Unweighted=3.1511 | Valid Loss Unweighted=3.2115 | Stale=0 | ETA=82.5m +rank0 [2026-10-28 12:22:49,968] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:22:50,025] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:22:50,028] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:22:50,189] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:22:51,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:22:52,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:22:52,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:22:53,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:22:53,691] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:22:53,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:22:53,748] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:22:53,751] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:22:53,863] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:22:55,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:22:55,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:22:56,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:22:57,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:22:57,216] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:22:57,220] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:22:57,273] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:22:57,276] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:22:57,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:22:58,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:22:59,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:22:59,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:23:00,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:23:00,569] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:23:00,576] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:23:00,838] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:23:01,246] INFO : Plotted number of elements +rank0 [2026-10-28 12:23:02,686] INFO : Plotted jets +rank0 [2026-10-28 12:23:03,774] INFO : Plotted jet ratio +rank0 [2026-10-28 12:23:03,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:23:04,036] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:23:04,427] INFO : Plotted number of elements +rank0 [2026-10-28 12:23:06,274] INFO : Plotted jets +rank0 [2026-10-28 12:23:07,374] INFO : Plotted jet ratio +rank0 [2026-10-28 12:23:07,406] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 12:23:07,612] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 12:23:07,976] INFO : Plotted number of elements +rank0 [2026-10-28 12:23:09,356] INFO : Plotted jets +rank0 [2026-10-28 12:23:10,414] INFO : Plotted jet ratio +rank0 [2026-10-28 12:23:21,446] INFO : Step 23100/50000 rank0 | Train Loss: 9.6468 | Train Loss Unweighted: 3.3150 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.9471 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1582 | Regression_eta: 0.0201 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7170 | Classification_binary: 2.9471 | Regression_cos_phi: 0.7279 | Regression_energy: 1.7298 | Regression_eta: 0.8972 | Regression_pt: 1.7866 | Regression_sin_phi: 0.8412 | LR: 6.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3585s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0497 | max_backward: 0.0329 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4571.1 | uss_MB: 4089.5 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:21,656] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:32,348] INFO : Step 23200/50000 rank0 | Train Loss: 8.9469 | Train Loss Unweighted: 3.3141 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.9570 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1575 | Regression_eta: 0.0103 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8189 | Classification_binary: 2.9570 | Regression_cos_phi: 0.4934 | Regression_energy: 1.7223 | Regression_eta: 0.4572 | Regression_pt: 1.8153 | Regression_sin_phi: 0.6827 | LR: 6.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0032 | max_backward: 0.0573 | max_optimizer_step: 0.1190 | max_tensorboard_logging: 0.2867 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4571.1 | uss_MB: 4089.6 | child_rss_MB: 3226.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:23:32,562] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:43,068] INFO : Step 23300/50000 rank0 | Train Loss: 8.5894 | Train Loss Unweighted: 3.1996 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.8411 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1622 | Regression_eta: 0.0069 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7537 | Classification_binary: 2.8411 | Regression_cos_phi: 0.4005 | Regression_energy: 1.7730 | Regression_eta: 0.3090 | Regression_pt: 1.8579 | Regression_sin_phi: 0.6542 | LR: 5.98e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2683s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0491 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3069 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4571.1 | uss_MB: 4089.7 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:43,283] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:54,049] INFO : Step 23400/50000 rank0 | Train Loss: 8.3101 | Train Loss Unweighted: 3.2046 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.8788 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1428 | Regression_eta: 0.0094 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7583 | Classification_binary: 2.8788 | Regression_cos_phi: 0.4622 | Regression_energy: 1.5607 | Regression_eta: 0.4187 | Regression_pt: 1.6689 | Regression_sin_phi: 0.5624 | LR: 5.95e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3291s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0163 | max_loss: 0.0382 | max_backward: 0.0381 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4571.1 | uss_MB: 4091.7 | child_rss_MB: 3201.5 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:23:54,261] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:05,280] INFO : Step 23500/50000 rank0 | Train Loss: 8.7530 | Train Loss Unweighted: 3.2836 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.9226 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1603 | Regression_eta: 0.0087 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8124 | Classification_binary: 2.9226 | Regression_cos_phi: 0.3815 | Regression_energy: 1.7525 | Regression_eta: 0.3876 | Regression_pt: 1.8774 | Regression_sin_phi: 0.6190 | LR: 5.92e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4104s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1050 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1194 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4571.1 | uss_MB: 4089.7 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:24:05,494] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:16,417] INFO : Step 23600/50000 rank0 | Train Loss: 8.7641 | Train Loss Unweighted: 3.1443 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7951 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1559 | Regression_eta: 0.0155 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7648 | Classification_binary: 2.7951 | Regression_cos_phi: 0.4720 | Regression_energy: 1.7046 | Regression_eta: 0.6912 | Regression_pt: 1.7055 | Regression_sin_phi: 0.6310 | LR: 5.90e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0455 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0344 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.3225 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4440.4 | uss_MB: 3958.0 | child_rss_MB: 3202.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:24:16,632] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:27,494] INFO : Step 23700/50000 rank0 | Train Loss: 8.9924 | Train Loss Unweighted: 3.2108 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8319 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1675 | Regression_eta: 0.0117 | Regression_pt: 0.1661 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7644 | Classification_binary: 2.8319 | Regression_cos_phi: 0.4139 | Regression_energy: 1.8309 | Regression_eta: 0.5222 | Regression_pt: 1.9732 | Regression_sin_phi: 0.6558 | LR: 5.87e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3146s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0903 | max_forward: 0.0180 | max_loss: 0.0032 | max_backward: 0.0384 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3066 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4440.4 | uss_MB: 3960.7 | child_rss_MB: 3205.0 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:24:27,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:38,560] INFO : Step 23800/50000 rank0 | Train Loss: 9.5199 | Train Loss Unweighted: 3.2353 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.8413 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1766 | Regression_eta: 0.0177 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7676 | Classification_binary: 2.8413 | Regression_cos_phi: 0.4759 | Regression_energy: 1.9310 | Regression_eta: 0.7908 | Regression_pt: 1.9460 | Regression_sin_phi: 0.7672 | LR: 5.84e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1146 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1175 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4440.4 | uss_MB: 3958.6 | child_rss_MB: 3208.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:24:38,772] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:49,687] INFO : Step 23900/50000 rank0 | Train Loss: 9.0106 | Train Loss Unweighted: 3.2836 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.9127 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1643 | Regression_eta: 0.0105 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7776 | Classification_binary: 2.9127 | Regression_cos_phi: 0.4456 | Regression_energy: 1.7965 | Regression_eta: 0.4703 | Regression_pt: 1.9147 | Regression_sin_phi: 0.6932 | LR: 5.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3131s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1065 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1179 | max_tensorboard_logging: 0.3110 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 4440.4 | uss_MB: 3958.7 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:24:49,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:00,449] INFO : Step 24000/50000 rank0 | Train Loss: 9.4634 | Train Loss Unweighted: 3.1679 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.7859 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1666 | Regression_eta: 0.0152 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.9082 | Classification_binary: 2.7859 | Regression_cos_phi: 0.5432 | Regression_energy: 1.8208 | Regression_eta: 0.6787 | Regression_pt: 1.8872 | Regression_sin_phi: 0.8393 | LR: 5.78e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2057s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0720 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.2704 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 4440.4 | uss_MB: 3958.4 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:25:00,662] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:00,681] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 12:25:00,762] INFO : Running validation on rank0 +rank0 [2026-10-28 12:25:00,762] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:25:00,762] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:25:01,957] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 12:25:02,103] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:25:02,235] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:25:02,309] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:25:02,378] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:25:02,446] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:25:02,516] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:25:02,587] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:25:02,696] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:25:02,829] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:25:02,961] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:25:03,033] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:25:03,102] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:25:03,171] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:25:03,241] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:25:03,311] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:25:03,414] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:25:03,421] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:25:03,424] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:25:03,425] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:25:03,428] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:25:03,430] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:25:03,431] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:25:03,434] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:25:03,437] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:25:03,440] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:25:03,441] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:25:03,444] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:25:03,446] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:25:03,447] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:25:03,450] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:25:03,452] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:25:09,320] INFO : VALIDATION | Step=24000/50000 | Train Loss=9.4634 | Valid Loss=9.0209 | Train Loss Unweighted=3.1679 | Valid Loss Unweighted=3.2470 | Stale=1 | ETA=78.6m +rank0 [2026-10-28 12:25:09,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:25:09,388] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:25:09,391] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:25:09,458] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:25:10,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:25:11,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:25:12,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:25:12,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:12,896] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:25:12,899] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:25:12,954] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:25:12,958] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:13,120] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:25:14,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:25:15,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:25:15,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:25:16,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:16,389] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:25:16,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:25:16,445] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:25:16,465] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:25:16,628] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:25:18,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:25:18,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:25:19,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:25:19,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:19,949] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:25:19,956] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:25:20,218] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:25:20,628] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:22,069] INFO : Plotted jets +rank0 [2026-10-28 12:25:23,156] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:23,188] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:23,417] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:23,811] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:25,706] INFO : Plotted jets +rank0 [2026-10-28 12:25:26,811] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:26,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 12:25:27,049] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 12:25:27,412] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:28,794] INFO : Plotted jets +rank0 [2026-10-28 12:25:29,842] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:40,995] INFO : Step 24100/50000 rank0 | Train Loss: 9.4241 | Train Loss Unweighted: 3.3252 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.9290 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1753 | Regression_eta: 0.0100 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7773 | Classification_binary: 2.9290 | Regression_cos_phi: 0.5549 | Regression_energy: 1.9164 | Regression_eta: 0.4454 | Regression_pt: 2.0666 | Regression_sin_phi: 0.7345 | LR: 5.75e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3890s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0927 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1190 | max_tensorboard_logging: 0.3819 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4632.0 | uss_MB: 4149.9 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:25:41,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:51,973] INFO : Step 24200/50000 rank0 | Train Loss: 9.3717 | Train Loss Unweighted: 3.2718 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8826 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1732 | Regression_eta: 0.0143 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7474 | Classification_binary: 2.8826 | Regression_cos_phi: 0.5631 | Regression_energy: 1.8938 | Regression_eta: 0.6372 | Regression_pt: 1.9722 | Regression_sin_phi: 0.6754 | LR: 5.73e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3970s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0862 | max_backward: 0.0332 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.3760 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4632.0 | uss_MB: 4149.9 | child_rss_MB: 3204.1 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:25:52,186] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:03,443] INFO : Step 24300/50000 rank0 | Train Loss: 9.0756 | Train Loss Unweighted: 3.2414 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.8783 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1605 | Regression_eta: 0.0124 | Regression_pt: 0.1544 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7296 | Classification_binary: 2.8783 | Regression_cos_phi: 0.4774 | Regression_energy: 1.7549 | Regression_eta: 0.5528 | Regression_pt: 1.8351 | Regression_sin_phi: 0.8475 | LR: 5.70e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3141s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.2288 | max_tensorboard_logging: 0.2810 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4632.0 | uss_MB: 4149.9 | child_rss_MB: 3201.9 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:26:03,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:14,303] INFO : Step 24400/50000 rank0 | Train Loss: 9.3651 | Train Loss Unweighted: 3.2259 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8516 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1620 | Regression_eta: 0.0202 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7842 | Classification_binary: 2.8516 | Regression_cos_phi: 0.4701 | Regression_energy: 1.7714 | Regression_eta: 0.8994 | Regression_pt: 1.8572 | Regression_sin_phi: 0.7313 | LR: 5.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3048s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0713 | max_forward: 0.0152 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3121 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4630.9 | uss_MB: 4149.7 | child_rss_MB: 3203.9 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:26:14,519] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:25,431] INFO : Step 24500/50000 rank0 | Train Loss: 9.2805 | Train Loss Unweighted: 3.1733 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.8069 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1592 | Regression_eta: 0.0139 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8133 | Classification_binary: 2.8069 | Regression_cos_phi: 0.6099 | Regression_energy: 1.7403 | Regression_eta: 0.6220 | Regression_pt: 1.8183 | Regression_sin_phi: 0.8698 | LR: 5.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3311s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0868 | max_forward: 0.0152 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3087 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4630.9 | uss_MB: 4149.8 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:26:25,643] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:36,495] INFO : Step 24600/50000 rank0 | Train Loss: 8.8440 | Train Loss Unweighted: 3.2523 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.9056 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1509 | Regression_eta: 0.0112 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7726 | Classification_binary: 2.9056 | Regression_cos_phi: 0.4615 | Regression_energy: 1.6494 | Regression_eta: 0.5013 | Regression_pt: 1.7657 | Regression_sin_phi: 0.7878 | LR: 5.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0468 | max_loss: 0.0031 | max_backward: 0.0385 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4510.3 | uss_MB: 4033.3 | child_rss_MB: 3202.4 | allocated_MB: 210.7 | reserved_MB: 2756.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:26:36,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:47,588] INFO : Step 24700/50000 rank0 | Train Loss: 9.5412 | Train Loss Unweighted: 3.1693 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.7823 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1690 | Regression_eta: 0.0150 | Regression_pt: 0.1619 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8576 | Classification_binary: 2.7823 | Regression_cos_phi: 0.6001 | Regression_energy: 1.8472 | Regression_eta: 0.6712 | Regression_pt: 1.9241 | Regression_sin_phi: 0.8587 | LR: 5.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3453s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0597 | max_loss: 0.0033 | max_backward: 0.0392 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4510.3 | uss_MB: 4033.7 | child_rss_MB: 3198.3 | allocated_MB: 210.7 | reserved_MB: 2756.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:26:47,805] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:58,677] INFO : Step 24800/50000 rank0 | Train Loss: 9.1292 | Train Loss Unweighted: 3.2207 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8518 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1627 | Regression_eta: 0.0092 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8064 | Classification_binary: 2.8518 | Regression_cos_phi: 0.4619 | Regression_energy: 1.7789 | Regression_eta: 0.4100 | Regression_pt: 1.8852 | Regression_sin_phi: 0.9350 | LR: 5.56e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2420s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0403 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1189 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4512.3 | uss_MB: 4032.4 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:26:58,891] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:09,697] INFO : Step 24900/50000 rank0 | Train Loss: 9.0073 | Train Loss Unweighted: 3.2081 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.8454 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1587 | Regression_eta: 0.0147 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6775 | Classification_binary: 2.8454 | Regression_cos_phi: 0.4963 | Regression_energy: 1.7350 | Regression_eta: 0.6562 | Regression_pt: 1.8502 | Regression_sin_phi: 0.7466 | LR: 5.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3435s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0237 | max_backward: 0.0329 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3994 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4512.3 | uss_MB: 4032.5 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:27:09,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:20,889] INFO : Step 25000/50000 rank0 | Train Loss: 9.1526 | Train Loss Unweighted: 3.2579 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8844 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1673 | Regression_eta: 0.0126 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7229 | Classification_binary: 2.8844 | Regression_cos_phi: 0.5866 | Regression_energy: 1.8289 | Regression_eta: 0.5622 | Regression_pt: 1.8784 | Regression_sin_phi: 0.6892 | LR: 5.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3232s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0419 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.3845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4513.3 | uss_MB: 4032.6 | child_rss_MB: 3198.8 | allocated_MB: 207.0 | reserved_MB: 2756.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:21,100] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:21,174] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 12:27:21,255] INFO : Running validation on rank0 +rank0 [2026-10-28 12:27:21,256] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:27:21,256] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:27:22,345] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 12:27:22,491] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:27:22,624] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:27:22,698] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:27:22,768] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:27:22,836] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:27:22,905] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:27:22,975] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:27:23,083] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:27:23,207] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:27:23,329] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:27:23,402] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:27:23,472] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:27:23,541] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:27:23,610] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:27:23,680] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:27:23,798] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:27:23,805] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:27:23,808] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:27:23,811] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:27:23,812] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:27:23,814] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:27:23,817] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:27:23,818] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:27:23,821] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:27:23,824] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:27:23,826] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:27:23,827] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:27:23,830] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:27:23,831] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:27:23,833] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:27:23,836] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:27:29,665] INFO : VALIDATION | Step=25000/50000 | Train Loss=9.1526 | Valid Loss=8.9001 | Train Loss Unweighted=3.2579 | Valid Loss Unweighted=3.2063 | Stale=0 | ETA=74.9m +rank0 [2026-10-28 12:27:29,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:27:29,730] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:27:29,733] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:27:29,892] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:27:31,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:27:32,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:27:32,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:27:33,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:27:33,637] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:27:33,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:27:33,695] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:27:33,709] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:33,880] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:27:35,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:27:35,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:27:36,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:27:37,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:27:37,126] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:27:37,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:27:37,182] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:27:37,194] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:27:37,355] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:27:38,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:27:39,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:27:39,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:27:40,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:27:40,674] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:27:40,681] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:27:40,944] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:27:41,352] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:42,795] INFO : Plotted jets +rank0 [2026-10-28 12:27:43,880] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:43,914] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:44,144] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:44,540] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:46,463] INFO : Plotted jets +rank0 [2026-10-28 12:27:47,572] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:47,617] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:47,829] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:48,192] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:49,579] INFO : Plotted jets +rank0 [2026-10-28 12:27:50,637] INFO : Plotted jet ratio +rank0 [2026-10-28 12:28:01,696] INFO : Step 25100/50000 rank0 | Train Loss: 8.2584 | Train Loss Unweighted: 3.0014 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.6806 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1374 | Regression_eta: 0.0114 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8410 | Classification_binary: 2.6806 | Regression_cos_phi: 0.4856 | Regression_energy: 1.5021 | Regression_eta: 0.5076 | Regression_pt: 1.6108 | Regression_sin_phi: 0.6308 | LR: 5.47e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0789 | max_backward: 0.0332 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3825 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4703.2 | uss_MB: 4226.0 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:01,907] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:12,894] INFO : Step 25200/50000 rank0 | Train Loss: 8.6830 | Train Loss Unweighted: 3.2349 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8900 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1529 | Regression_eta: 0.0110 | Regression_pt: 0.1471 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7216 | Classification_binary: 2.8900 | Regression_cos_phi: 0.5159 | Regression_energy: 1.6715 | Regression_eta: 0.4886 | Regression_pt: 1.7477 | Regression_sin_phi: 0.6477 | LR: 5.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3405s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0367 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3770 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4703.2 | uss_MB: 4228.2 | child_rss_MB: 3196.5 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:28:13,108] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:23,966] INFO : Step 25300/50000 rank0 | Train Loss: 9.2545 | Train Loss Unweighted: 3.1495 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7586 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1750 | Regression_eta: 0.0125 | Regression_pt: 0.1677 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7309 | Classification_binary: 2.7586 | Regression_cos_phi: 0.5234 | Regression_energy: 1.9128 | Regression_eta: 0.5563 | Regression_pt: 1.9929 | Regression_sin_phi: 0.7796 | LR: 5.42e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3667s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0997 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3734 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4703.2 | uss_MB: 4226.1 | child_rss_MB: 3193.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:28:24,182] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:34,753] INFO : Step 25400/50000 rank0 | Train Loss: 8.6752 | Train Loss Unweighted: 3.2080 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.8626 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1555 | Regression_eta: 0.0106 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7418 | Classification_binary: 2.8626 | Regression_cos_phi: 0.4412 | Regression_energy: 1.7003 | Regression_eta: 0.4726 | Regression_pt: 1.7237 | Regression_sin_phi: 0.7330 | LR: 5.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2264s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0785 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.2724 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4703.2 | uss_MB: 4226.0 | child_rss_MB: 3215.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:34,967] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:45,775] INFO : Step 25500/50000 rank0 | Train Loss: 8.9248 | Train Loss Unweighted: 3.1332 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.7742 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1571 | Regression_eta: 0.0143 | Regression_pt: 0.1513 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7880 | Classification_binary: 2.7742 | Regression_cos_phi: 0.5333 | Regression_energy: 1.7173 | Regression_eta: 0.6357 | Regression_pt: 1.7977 | Regression_sin_phi: 0.6786 | LR: 5.36e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2894s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4703.2 | uss_MB: 4226.2 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:46,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:56,923] INFO : Step 25600/50000 rank0 | Train Loss: 8.1805 | Train Loss Unweighted: 3.0910 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7747 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1393 | Regression_eta: 0.0112 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7115 | Classification_binary: 2.7747 | Regression_cos_phi: 0.5315 | Regression_energy: 1.5230 | Regression_eta: 0.4974 | Regression_pt: 1.5779 | Regression_sin_phi: 0.5646 | LR: 5.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2689s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.2763 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 4583.5 | uss_MB: 4106.1 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:28:57,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:08,075] INFO : Step 25700/50000 rank0 | Train Loss: 9.5677 | Train Loss Unweighted: 3.2010 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.8236 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1667 | Regression_eta: 0.0250 | Regression_pt: 0.1498 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7357 | Classification_binary: 2.8236 | Regression_cos_phi: 0.5658 | Regression_energy: 1.8227 | Regression_eta: 1.1150 | Regression_pt: 1.7794 | Regression_sin_phi: 0.7255 | LR: 5.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2719s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0340 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.2714 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4583.5 | uss_MB: 4106.2 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:29:08,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:19,223] INFO : Step 25800/50000 rank0 | Train Loss: 8.6022 | Train Loss Unweighted: 3.1731 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8376 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1474 | Regression_eta: 0.0119 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7261 | Classification_binary: 2.8376 | Regression_cos_phi: 0.4433 | Regression_energy: 1.6111 | Regression_eta: 0.5329 | Regression_pt: 1.6867 | Regression_sin_phi: 0.7644 | LR: 5.27e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2550s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0344 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.2726 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4584.5 | uss_MB: 4106.8 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:29:19,451] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:30,128] INFO : Step 25900/50000 rank0 | Train Loss: 9.0806 | Train Loss Unweighted: 3.2974 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.9409 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1559 | Regression_eta: 0.0112 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7751 | Classification_binary: 2.9409 | Regression_cos_phi: 0.5958 | Regression_energy: 1.7041 | Regression_eta: 0.4979 | Regression_pt: 1.8012 | Regression_sin_phi: 0.7656 | LR: 5.25e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3061s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0691 | max_forward: 0.0145 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3178 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4584.5 | uss_MB: 4106.8 | child_rss_MB: 3197.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:29:30,343] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:41,151] INFO : Step 26000/50000 rank0 | Train Loss: 8.5188 | Train Loss Unweighted: 3.0692 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7322 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1482 | Regression_eta: 0.0131 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8342 | Classification_binary: 2.7322 | Regression_cos_phi: 0.4715 | Regression_energy: 1.6199 | Regression_eta: 0.5842 | Regression_pt: 1.6616 | Regression_sin_phi: 0.6154 | LR: 5.22e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0214 | max_backward: 0.0344 | max_optimizer_step: 0.1205 | max_tensorboard_logging: 0.3825 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4584.5 | uss_MB: 4106.8 | child_rss_MB: 3198.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:29:41,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:41,384] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 12:29:41,573] INFO : Running validation on rank0 +rank0 [2026-10-28 12:29:41,573] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:29:41,573] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:29:42,659] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 12:29:42,805] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:29:42,939] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:29:43,014] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:29:43,087] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:29:43,156] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:29:43,227] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:29:43,297] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:29:43,404] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:29:43,536] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:29:43,668] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:29:43,742] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:29:43,814] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:29:43,883] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:29:43,954] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:29:44,024] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:29:44,128] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:29:44,135] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:29:44,138] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:29:44,141] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:29:44,142] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:29:44,144] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:29:44,147] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:29:44,148] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:29:44,151] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:29:44,154] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:29:44,157] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:29:44,159] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:29:44,160] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:29:44,163] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:29:44,165] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:29:44,166] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:29:50,222] INFO : VALIDATION | Step=26000/50000 | Train Loss=8.5188 | Valid Loss=8.6956 | Train Loss Unweighted=3.0692 | Valid Loss Unweighted=3.1324 | Stale=0 | ETA=71.3m +rank0 [2026-10-28 12:29:50,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:29:50,288] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:29:50,291] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:29:50,458] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:29:52,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:29:52,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:29:53,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:29:53,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:29:53,884] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:29:53,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:29:53,942] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:29:53,944] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:29:54,046] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:29:55,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:29:56,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:29:56,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:29:57,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:29:57,509] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:29:57,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:29:57,566] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:29:57,569] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:29:57,676] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:29:59,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:29:59,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:30:00,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:30:00,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:30:00,945] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:30:00,952] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:30:01,219] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:30:01,630] INFO : Plotted number of elements +rank0 [2026-10-28 12:30:03,077] INFO : Plotted jets +rank0 [2026-10-28 12:30:04,170] INFO : Plotted jet ratio +rank0 [2026-10-28 12:30:04,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:30:04,439] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:30:04,830] INFO : Plotted number of elements +rank0 [2026-10-28 12:30:06,703] INFO : Plotted jets +rank0 [2026-10-28 12:30:07,803] INFO : Plotted jet ratio +rank0 [2026-10-28 12:30:07,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 12:30:08,048] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 12:30:08,411] INFO : Plotted number of elements +rank0 [2026-10-28 12:30:09,790] INFO : Plotted jets +rank0 [2026-10-28 12:30:10,840] INFO : Plotted jet ratio +rank0 [2026-10-28 12:30:22,007] INFO : Step 26100/50000 rank0 | Train Loss: 8.7575 | Train Loss Unweighted: 3.2024 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8523 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1534 | Regression_eta: 0.0110 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7623 | Classification_binary: 2.8523 | Regression_cos_phi: 0.4162 | Regression_energy: 1.6772 | Regression_eta: 0.4896 | Regression_pt: 1.7933 | Regression_sin_phi: 0.7666 | LR: 5.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2547s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0828 | max_optimizer_step: 0.1207 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4773.1 | uss_MB: 4296.6 | child_rss_MB: 3219.8 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:22,222] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:33,164] INFO : Step 26200/50000 rank0 | Train Loss: 8.4734 | Train Loss Unweighted: 3.0377 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6827 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1588 | Regression_eta: 0.0102 | Regression_pt: 0.1543 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6693 | Classification_binary: 2.6827 | Regression_cos_phi: 0.4549 | Regression_energy: 1.7365 | Regression_eta: 0.4569 | Regression_pt: 1.8340 | Regression_sin_phi: 0.6391 | LR: 5.16e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0189 | max_loss: 0.0034 | max_backward: 0.0616 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4773.1 | uss_MB: 4298.7 | child_rss_MB: 3220.1 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:30:33,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:44,167] INFO : Step 26300/50000 rank0 | Train Loss: 9.2084 | Train Loss Unweighted: 3.1564 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7777 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1683 | Regression_eta: 0.0161 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7111 | Classification_binary: 2.7777 | Regression_cos_phi: 0.5491 | Regression_energy: 1.8404 | Regression_eta: 0.7194 | Regression_pt: 1.8926 | Regression_sin_phi: 0.7181 | LR: 5.13e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3073s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4773.1 | uss_MB: 4296.7 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:30:44,391] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:55,034] INFO : Step 26400/50000 rank0 | Train Loss: 8.4618 | Train Loss Unweighted: 3.1612 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.8185 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1494 | Regression_eta: 0.0114 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8176 | Classification_binary: 2.8185 | Regression_cos_phi: 0.3808 | Regression_energy: 1.6334 | Regression_eta: 0.5072 | Regression_pt: 1.7657 | Regression_sin_phi: 0.5385 | LR: 5.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2688s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0151 | max_forward: 0.0143 | max_loss: 0.0032 | max_backward: 0.0327 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.3066 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4773.1 | uss_MB: 4296.8 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:55,249] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:05,995] INFO : Step 26500/50000 rank0 | Train Loss: 9.1826 | Train Loss Unweighted: 3.1310 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7751 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1527 | Regression_eta: 0.0213 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7348 | Classification_binary: 2.7751 | Regression_cos_phi: 0.5025 | Regression_energy: 1.6696 | Regression_eta: 0.9495 | Regression_pt: 1.7343 | Regression_sin_phi: 0.8168 | LR: 5.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4090s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0169 | max_loss: 0.0821 | max_backward: 0.0382 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3855 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4773.1 | uss_MB: 4298.8 | child_rss_MB: 3197.2 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:31:06,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:17,192] INFO : Step 26600/50000 rank0 | Train Loss: 8.8570 | Train Loss Unweighted: 3.1231 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7779 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1510 | Regression_eta: 0.0146 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7589 | Classification_binary: 2.7779 | Regression_cos_phi: 0.5587 | Regression_energy: 1.6511 | Regression_eta: 0.6522 | Regression_pt: 1.6960 | Regression_sin_phi: 0.7622 | LR: 5.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0684 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3979 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4655.2 | uss_MB: 4178.7 | child_rss_MB: 3200.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:17,404] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:28,313] INFO : Step 26700/50000 rank0 | Train Loss: 9.5326 | Train Loss Unweighted: 3.2361 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.8431 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1746 | Regression_eta: 0.0164 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8320 | Classification_binary: 2.8431 | Regression_cos_phi: 0.5391 | Regression_energy: 1.9091 | Regression_eta: 0.7300 | Regression_pt: 1.9476 | Regression_sin_phi: 0.7317 | LR: 5.02e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0814 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1186 | max_tensorboard_logging: 0.3166 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4655.2 | uss_MB: 4179.8 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:28,528] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:39,238] INFO : Step 26800/50000 rank0 | Train Loss: 8.1123 | Train Loss Unweighted: 3.0347 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.7095 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1446 | Regression_eta: 0.0097 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7193 | Classification_binary: 2.7095 | Regression_cos_phi: 0.4018 | Regression_energy: 1.5809 | Regression_eta: 0.4304 | Regression_pt: 1.6531 | Regression_sin_phi: 0.6173 | LR: 4.99e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3805s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0836 | max_backward: 0.0329 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4655.2 | uss_MB: 4180.0 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:39,448] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:50,323] INFO : Step 26900/50000 rank0 | Train Loss: 8.9591 | Train Loss Unweighted: 3.1512 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.7565 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1777 | Regression_eta: 0.0092 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8104 | Classification_binary: 2.7565 | Regression_cos_phi: 0.4106 | Regression_energy: 1.9432 | Regression_eta: 0.4115 | Regression_pt: 2.0664 | Regression_sin_phi: 0.5605 | LR: 4.96e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0870 | max_backward: 0.0328 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.3701 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4655.2 | uss_MB: 4179.0 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:31:50,536] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:01,233] INFO : Step 27000/50000 rank0 | Train Loss: 8.5175 | Train Loss Unweighted: 3.1232 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7843 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1484 | Regression_eta: 0.0100 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7345 | Classification_binary: 2.7843 | Regression_cos_phi: 0.5530 | Regression_energy: 1.6225 | Regression_eta: 0.4469 | Regression_pt: 1.7311 | Regression_sin_phi: 0.6451 | LR: 4.94e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.2192s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0032 | max_backward: 0.0485 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4656.2 | uss_MB: 4180.1 | child_rss_MB: 3217.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:01,445] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:01,464] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 12:32:01,545] INFO : Running validation on rank0 +rank0 [2026-10-28 12:32:01,545] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:32:01,545] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:32:02,759] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 12:32:02,905] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:32:03,036] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:32:03,110] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:32:03,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:32:03,246] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:32:03,316] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:32:03,387] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:32:03,496] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:32:03,628] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:32:03,759] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:32:03,829] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:32:03,899] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:32:03,968] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:32:04,040] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:32:04,113] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:32:04,230] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:32:04,238] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:32:04,240] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:32:04,243] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:32:04,244] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:32:04,247] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:32:04,250] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:32:04,251] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:32:04,254] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:32:04,257] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:32:04,257] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:32:04,260] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:32:04,261] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:32:04,264] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:32:04,266] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:32:04,267] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:32:10,079] INFO : VALIDATION | Step=27000/50000 | Train Loss=8.5175 | Valid Loss=8.8705 | Train Loss Unweighted=3.1232 | Valid Loss Unweighted=3.1739 | Stale=1 | ETA=67.8m +rank0 [2026-10-28 12:32:10,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:10,155] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:32:10,158] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:32:10,321] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:11,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:12,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:13,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:13,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:13,710] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:32:13,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:13,769] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:32:13,772] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:13,877] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:15,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:15,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:16,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:17,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:17,219] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:32:17,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:17,276] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:32:17,279] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:32:17,393] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:18,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:19,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:19,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:20,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:20,461] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:32:20,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:32:20,730] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:32:21,140] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:22,579] INFO : Plotted jets +rank0 [2026-10-28 12:32:23,662] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:23,693] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:23,928] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:24,319] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:26,193] INFO : Plotted jets +rank0 [2026-10-28 12:32:27,294] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:27,326] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 12:32:27,533] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 12:32:27,899] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:29,372] INFO : Plotted jets +rank0 [2026-10-28 12:32:30,430] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:41,586] INFO : Step 27100/50000 rank0 | Train Loss: 8.6993 | Train Loss Unweighted: 3.0721 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7215 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1543 | Regression_eta: 0.0118 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7528 | Classification_binary: 2.7215 | Regression_cos_phi: 0.4793 | Regression_energy: 1.6865 | Regression_eta: 0.5247 | Regression_pt: 1.7713 | Regression_sin_phi: 0.7632 | LR: 4.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4073s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1012 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4852.7 | uss_MB: 4377.1 | child_rss_MB: 3196.2 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:32:41,799] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:52,587] INFO : Step 27200/50000 rank0 | Train Loss: 8.2814 | Train Loss Unweighted: 3.0758 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.7533 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1407 | Regression_eta: 0.0112 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6688 | Classification_binary: 2.7533 | Regression_cos_phi: 0.4863 | Regression_energy: 1.5385 | Regression_eta: 0.5011 | Regression_pt: 1.6394 | Regression_sin_phi: 0.6940 | LR: 4.88e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3757s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0577 | max_backward: 0.0341 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3745 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4852.7 | uss_MB: 4377.1 | child_rss_MB: 3197.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:32:52,804] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:03,690] INFO : Step 27300/50000 rank0 | Train Loss: 9.0032 | Train Loss Unweighted: 3.2906 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.9190 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1670 | Regression_eta: 0.0160 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7181 | Classification_binary: 2.9190 | Regression_cos_phi: 0.4247 | Regression_energy: 1.8260 | Regression_eta: 0.7114 | Regression_pt: 1.8689 | Regression_sin_phi: 0.5349 | LR: 4.85e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4052s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0982 | max_backward: 0.0335 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4852.7 | uss_MB: 4377.1 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:33:03,901] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:14,887] INFO : Step 27400/50000 rank0 | Train Loss: 8.3476 | Train Loss Unweighted: 3.0878 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7612 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1428 | Regression_eta: 0.0109 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7627 | Classification_binary: 2.7612 | Regression_cos_phi: 0.4945 | Regression_energy: 1.5616 | Regression_eta: 0.4841 | Regression_pt: 1.6436 | Regression_sin_phi: 0.6399 | LR: 4.82e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1214 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4852.7 | uss_MB: 4377.2 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:33:15,102] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:25,989] INFO : Step 27500/50000 rank0 | Train Loss: 8.9350 | Train Loss Unweighted: 3.1282 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7858 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1473 | Regression_eta: 0.0160 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7247 | Classification_binary: 2.7858 | Regression_cos_phi: 0.5819 | Regression_energy: 1.6108 | Regression_eta: 0.7155 | Regression_pt: 1.6868 | Regression_sin_phi: 0.8295 | LR: 4.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3617s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0468 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3795 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4852.7 | uss_MB: 4377.3 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:33:26,204] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:37,118] INFO : Step 27600/50000 rank0 | Train Loss: 8.0661 | Train Loss Unweighted: 2.9602 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.6369 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1425 | Regression_eta: 0.0084 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7787 | Classification_binary: 2.6369 | Regression_cos_phi: 0.4724 | Regression_energy: 1.5578 | Regression_eta: 0.3745 | Regression_pt: 1.6401 | Regression_sin_phi: 0.6057 | LR: 4.77e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4227s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1005 | max_loss: 0.0031 | max_backward: 0.0331 | max_optimizer_step: 0.1211 | max_tensorboard_logging: 0.3717 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4852.7 | uss_MB: 4377.3 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:33:37,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:48,221] INFO : Step 27700/50000 rank0 | Train Loss: 7.9794 | Train Loss Unweighted: 2.9328 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6192 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1364 | Regression_eta: 0.0104 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7312 | Classification_binary: 2.6192 | Regression_cos_phi: 0.4545 | Regression_energy: 1.4914 | Regression_eta: 0.4656 | Regression_pt: 1.5870 | Regression_sin_phi: 0.6305 | LR: 4.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4247s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1205 | max_loss: 0.0031 | max_backward: 0.0342 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3809 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4728.8 | uss_MB: 4254.5 | child_rss_MB: 3194.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:48,439] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:33:59,236] INFO : Step 27800/50000 rank0 | Train Loss: 8.6637 | Train Loss Unweighted: 3.1647 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8083 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1576 | Regression_eta: 0.0108 | Regression_pt: 0.1549 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7473 | Classification_binary: 2.8083 | Regression_cos_phi: 0.4251 | Regression_energy: 1.7225 | Regression_eta: 0.4800 | Regression_pt: 1.8407 | Regression_sin_phi: 0.6397 | LR: 4.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.1048 | max_backward: 0.0336 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3749 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4728.8 | uss_MB: 4254.6 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:59,449] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:10,131] INFO : Step 27900/50000 rank0 | Train Loss: 8.0670 | Train Loss Unweighted: 2.9797 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.6609 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1438 | Regression_eta: 0.0094 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7385 | Classification_binary: 2.6609 | Regression_cos_phi: 0.4685 | Regression_energy: 1.5726 | Regression_eta: 0.4180 | Regression_pt: 1.5682 | Regression_sin_phi: 0.6402 | LR: 4.68e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2745s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0033 | max_backward: 0.0905 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2833 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4728.8 | uss_MB: 4254.6 | child_rss_MB: 3221.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:34:10,344] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:21,244] INFO : Step 28000/50000 rank0 | Train Loss: 8.6490 | Train Loss Unweighted: 3.1135 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.7583 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1578 | Regression_eta: 0.0107 | Regression_pt: 0.1522 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 2.7583 | Regression_cos_phi: 0.4548 | Regression_energy: 1.7247 | Regression_eta: 0.4767 | Regression_pt: 1.8079 | Regression_sin_phi: 0.6418 | LR: 4.66e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3279s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0032 | max_backward: 0.1418 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2740 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4728.8 | uss_MB: 4254.7 | child_rss_MB: 3220.4 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:34:21,458] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:21,477] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 12:34:21,560] INFO : Running validation on rank0 +rank0 [2026-10-28 12:34:21,560] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:34:21,560] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:34:22,757] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 12:34:22,902] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:34:23,035] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:34:23,109] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:34:23,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:34:23,247] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:34:23,316] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:34:23,388] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:34:23,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:34:23,629] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:34:23,760] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:34:23,832] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:34:23,901] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:34:23,969] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:34:24,039] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:34:24,111] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:34:24,217] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:34:24,224] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:34:24,226] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:34:24,227] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:34:24,230] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:34:24,233] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:34:24,234] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:34:24,236] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:34:24,238] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:34:24,240] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:34:24,243] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:34:24,246] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:34:24,247] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:34:24,249] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:34:24,252] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:34:24,253] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:34:30,108] INFO : VALIDATION | Step=28000/50000 | Train Loss=8.6490 | Valid Loss=8.7492 | Train Loss Unweighted=3.1135 | Valid Loss Unweighted=3.1402 | Stale=2 | ETA=64.4m +rank0 [2026-10-28 12:34:30,118] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:30,174] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:34:30,178] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:30,275] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:31,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:32,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:32,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:33,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:33,765] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:34:33,769] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:33,835] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:34:33,838] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:33,932] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:35,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:36,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:36,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:37,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:37,318] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:34:37,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:37,376] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:34:37,378] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:34:37,484] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:39,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:39,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:40,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:40,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:40,802] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:34:40,810] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:41,075] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:41,484] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:42,918] INFO : Plotted jets +rank0 [2026-10-28 12:34:44,000] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:44,032] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:44,267] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:44,662] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:46,585] INFO : Plotted jets +rank0 [2026-10-28 12:34:47,690] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:47,722] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:47,935] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:48,299] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:49,674] INFO : Plotted jets +rank0 [2026-10-28 12:34:50,729] INFO : Plotted jet ratio +rank0 [2026-10-28 12:35:01,940] INFO : Step 28100/50000 rank0 | Train Loss: 8.4413 | Train Loss Unweighted: 3.0659 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7427 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1400 | Regression_eta: 0.0146 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7446 | Classification_binary: 2.7427 | Regression_cos_phi: 0.3990 | Regression_energy: 1.5304 | Regression_eta: 0.6524 | Regression_pt: 1.5978 | Regression_sin_phi: 0.7744 | LR: 4.63e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4502s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1242 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4928.2 | uss_MB: 4453.9 | child_rss_MB: 3198.9 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:35:02,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:13,008] INFO : Step 28200/50000 rank0 | Train Loss: 8.8933 | Train Loss Unweighted: 3.0885 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.7558 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1397 | Regression_eta: 0.0175 | Regression_pt: 0.1358 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8403 | Classification_binary: 2.7558 | Regression_cos_phi: 0.5929 | Regression_energy: 1.5271 | Regression_eta: 0.7783 | Regression_pt: 1.6136 | Regression_sin_phi: 0.7853 | LR: 4.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0166 | max_loss: 0.0425 | max_backward: 0.0382 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.4274 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4928.2 | uss_MB: 4456.0 | child_rss_MB: 3199.3 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:35:13,219] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:24,198] INFO : Step 28300/50000 rank0 | Train Loss: 9.2510 | Train Loss Unweighted: 3.1837 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.8101 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1647 | Regression_eta: 0.0172 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7424 | Classification_binary: 2.8101 | Regression_cos_phi: 0.4889 | Regression_energy: 1.8004 | Regression_eta: 0.7686 | Regression_pt: 1.8556 | Regression_sin_phi: 0.7850 | LR: 4.57e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4577s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0984 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.3918 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4928.2 | uss_MB: 4454.0 | child_rss_MB: 3197.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:35:24,413] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:35,287] INFO : Step 28400/50000 rank0 | Train Loss: 8.4454 | Train Loss Unweighted: 3.0744 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7281 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1528 | Regression_eta: 0.0097 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7118 | Classification_binary: 2.7281 | Regression_cos_phi: 0.4346 | Regression_energy: 1.6710 | Regression_eta: 0.4321 | Regression_pt: 1.7947 | Regression_sin_phi: 0.6731 | LR: 4.55e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2806s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0279 | max_forward: 0.0167 | max_loss: 0.0037 | max_backward: 0.0342 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4928.2 | uss_MB: 4454.0 | child_rss_MB: 3203.6 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:35:35,499] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:46,378] INFO : Step 28500/50000 rank0 | Train Loss: 8.7826 | Train Loss Unweighted: 3.0968 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.7558 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1469 | Regression_eta: 0.0175 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6823 | Classification_binary: 2.7558 | Regression_cos_phi: 0.5985 | Regression_energy: 1.6060 | Regression_eta: 0.7798 | Regression_pt: 1.6890 | Regression_sin_phi: 0.6711 | LR: 4.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3950s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0792 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3803 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4928.2 | uss_MB: 4454.1 | child_rss_MB: 3199.6 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:35:46,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:57,416] INFO : Step 28600/50000 rank0 | Train Loss: 8.4523 | Train Loss Unweighted: 3.0593 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.7246 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1474 | Regression_eta: 0.0111 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6942 | Classification_binary: 2.7246 | Regression_cos_phi: 0.4386 | Regression_energy: 1.6112 | Regression_eta: 0.4940 | Regression_pt: 1.6939 | Regression_sin_phi: 0.7957 | LR: 4.49e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4280s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0573 | max_backward: 0.0335 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3885 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4928.2 | uss_MB: 4454.0 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:35:57,631] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:08,599] INFO : Step 28700/50000 rank0 | Train Loss: 8.4228 | Train Loss Unweighted: 3.0729 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.7447 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1423 | Regression_eta: 0.0152 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7434 | Classification_binary: 2.7447 | Regression_cos_phi: 0.4148 | Regression_energy: 1.5561 | Regression_eta: 0.6795 | Regression_pt: 1.6350 | Regression_sin_phi: 0.6493 | LR: 4.46e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4101s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1006 | max_loss: 0.0036 | max_backward: 0.0328 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3821 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4803.7 | uss_MB: 4330.4 | child_rss_MB: 3201.4 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:36:08,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:19,703] INFO : Step 28800/50000 rank0 | Train Loss: 8.8311 | Train Loss Unweighted: 3.1146 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7493 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1622 | Regression_eta: 0.0128 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7268 | Classification_binary: 2.7493 | Regression_cos_phi: 0.4707 | Regression_energy: 1.7734 | Regression_eta: 0.5709 | Regression_pt: 1.8601 | Regression_sin_phi: 0.6799 | LR: 4.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4106s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0935 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1217 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4803.7 | uss_MB: 4330.5 | child_rss_MB: 3197.2 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:36:19,918] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:30,505] INFO : Step 28900/50000 rank0 | Train Loss: 9.2975 | Train Loss Unweighted: 3.2516 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8664 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1722 | Regression_eta: 0.0141 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7252 | Classification_binary: 2.8664 | Regression_cos_phi: 0.5357 | Regression_energy: 1.8830 | Regression_eta: 0.6305 | Regression_pt: 1.9457 | Regression_sin_phi: 0.7109 | LR: 4.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0990 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4803.7 | uss_MB: 4330.5 | child_rss_MB: 3226.5 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:36:30,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:41,611] INFO : Step 29000/50000 rank0 | Train Loss: 8.6444 | Train Loss Unweighted: 3.1025 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.7444 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1584 | Regression_eta: 0.0098 | Regression_pt: 0.1555 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7700 | Classification_binary: 2.7444 | Regression_cos_phi: 0.4613 | Regression_energy: 1.7314 | Regression_eta: 0.4361 | Regression_pt: 1.8479 | Regression_sin_phi: 0.6533 | LR: 4.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0165 | max_loss: 0.0034 | max_backward: 0.1450 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4803.7 | uss_MB: 4333.1 | child_rss_MB: 3225.7 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:36:41,825] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:41,843] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 12:36:41,926] INFO : Running validation on rank0 +rank0 [2026-10-28 12:36:41,926] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:36:41,927] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:36:43,144] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 12:36:43,289] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:36:43,421] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:36:43,495] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:36:43,564] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:36:43,632] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:36:43,704] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:36:43,774] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:36:43,883] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:36:44,015] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:36:44,145] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:36:44,217] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:36:44,286] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:36:44,355] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:36:44,425] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:36:44,495] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:36:44,599] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:36:44,606] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:36:44,609] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:36:44,612] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:36:44,614] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:36:44,616] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:36:44,620] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:36:44,622] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:36:44,623] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:36:44,626] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:36:44,629] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:36:44,630] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:36:44,632] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:36:44,635] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:36:44,636] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:36:44,639] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:36:50,505] INFO : VALIDATION | Step=29000/50000 | Train Loss=8.6444 | Valid Loss=8.4959 | Train Loss Unweighted=3.1025 | Valid Loss Unweighted=3.0585 | Stale=0 | ETA=61.0m +rank0 [2026-10-28 12:36:50,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:36:50,572] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:36:50,576] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:36:50,745] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:36:52,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:36:52,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:36:53,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:36:54,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:36:54,162] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:36:54,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:36:54,220] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:36:54,223] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:36:54,334] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:36:55,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:36:56,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:36:56,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:36:57,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:36:57,554] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:36:57,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:36:57,611] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:36:57,614] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:36:57,781] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:36:59,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:36:59,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:37:00,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:37:00,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:37:00,977] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:37:00,985] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:37:01,242] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:37:01,651] INFO : Plotted number of elements +rank0 [2026-10-28 12:37:03,099] INFO : Plotted jets +rank0 [2026-10-28 12:37:04,185] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:04,221] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:37:04,460] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:37:04,852] INFO : Plotted number of elements +rank0 [2026-10-28 12:37:06,736] INFO : Plotted jets +rank0 [2026-10-28 12:37:07,843] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:07,877] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 12:37:08,085] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 12:37:08,454] INFO : Plotted number of elements +rank0 [2026-10-28 12:37:09,836] INFO : Plotted jets +rank0 [2026-10-28 12:37:10,897] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:22,135] INFO : Step 29100/50000 rank0 | Train Loss: 8.1597 | Train Loss Unweighted: 3.0292 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7121 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1384 | Regression_eta: 0.0106 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7109 | Classification_binary: 2.7121 | Regression_cos_phi: 0.4648 | Regression_energy: 1.5132 | Regression_eta: 0.4743 | Regression_pt: 1.6018 | Regression_sin_phi: 0.6825 | LR: 4.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4221s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0158 | max_loss: 0.0898 | max_backward: 0.0338 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4996.4 | uss_MB: 4523.5 | child_rss_MB: 3205.3 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:37:22,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:33,371] INFO : Step 29200/50000 rank0 | Train Loss: 8.4643 | Train Loss Unweighted: 3.1143 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7650 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1568 | Regression_eta: 0.0100 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7246 | Classification_binary: 2.7650 | Regression_cos_phi: 0.4444 | Regression_energy: 1.7140 | Regression_eta: 0.4474 | Regression_pt: 1.7846 | Regression_sin_phi: 0.5842 | LR: 4.33e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0185 | max_forward: 0.0146 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2994 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4996.4 | uss_MB: 4523.5 | child_rss_MB: 3207.3 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:33,584] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:44,438] INFO : Step 29300/50000 rank0 | Train Loss: 8.4119 | Train Loss Unweighted: 3.0736 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7302 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1512 | Regression_eta: 0.0107 | Regression_pt: 0.1489 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7303 | Classification_binary: 2.7302 | Regression_cos_phi: 0.4200 | Regression_energy: 1.6528 | Regression_eta: 0.4776 | Regression_pt: 1.7696 | Regression_sin_phi: 0.6314 | LR: 4.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3558s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0427 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.3764 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:44,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:55,556] INFO : Step 29400/50000 rank0 | Train Loss: 8.3027 | Train Loss Unweighted: 3.0531 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7216 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1449 | Regression_eta: 0.0084 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7606 | Classification_binary: 2.7216 | Regression_cos_phi: 0.5280 | Regression_energy: 1.5837 | Regression_eta: 0.3729 | Regression_pt: 1.7009 | Regression_sin_phi: 0.6349 | LR: 4.27e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4414s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1097 | max_loss: 0.0035 | max_backward: 0.0342 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3203.6 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:55,768] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:06,648] INFO : Step 29500/50000 rank0 | Train Loss: 8.2841 | Train Loss Unweighted: 2.9511 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6207 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1439 | Regression_eta: 0.0090 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7643 | Classification_binary: 2.6207 | Regression_cos_phi: 0.4873 | Regression_energy: 1.5736 | Regression_eta: 0.4000 | Regression_pt: 1.6838 | Regression_sin_phi: 0.7544 | LR: 4.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3404s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1056 | max_forward: 0.0148 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:38:06,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:17,600] INFO : Step 29600/50000 rank0 | Train Loss: 8.3190 | Train Loss Unweighted: 3.0665 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7338 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1463 | Regression_eta: 0.0101 | Regression_pt: 0.1425 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7529 | Classification_binary: 2.7338 | Regression_cos_phi: 0.4659 | Regression_energy: 1.5993 | Regression_eta: 0.4520 | Regression_pt: 1.6934 | Regression_sin_phi: 0.6218 | LR: 4.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3825s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0747 | max_backward: 0.0332 | max_optimizer_step: 0.1215 | max_tensorboard_logging: 0.3699 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4996.4 | uss_MB: 4523.6 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:38:17,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:28,820] INFO : Step 29700/50000 rank0 | Train Loss: 9.1576 | Train Loss Unweighted: 3.2718 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8855 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1727 | Regression_eta: 0.0118 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7282 | Classification_binary: 2.8855 | Regression_cos_phi: 0.4942 | Regression_energy: 1.8878 | Regression_eta: 0.5250 | Regression_pt: 1.9980 | Regression_sin_phi: 0.6389 | LR: 4.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0307 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1228 | max_tensorboard_logging: 0.3704 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4875.3 | uss_MB: 4404.7 | child_rss_MB: 3202.9 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:38:29,036] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:39,779] INFO : Step 29800/50000 rank0 | Train Loss: 7.7644 | Train Loss Unweighted: 3.0393 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7267 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1390 | Regression_eta: 0.0076 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7120 | Classification_binary: 2.7267 | Regression_cos_phi: 0.3251 | Regression_energy: 1.5191 | Regression_eta: 0.3368 | Regression_pt: 1.6242 | Regression_sin_phi: 0.5204 | LR: 4.16e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3984s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0900 | max_backward: 0.0334 | max_optimizer_step: 0.1219 | max_tensorboard_logging: 0.3746 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4875.3 | uss_MB: 4403.0 | child_rss_MB: 3200.5 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:38:39,992] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:51,024] INFO : Step 29900/50000 rank0 | Train Loss: 9.5750 | Train Loss Unweighted: 3.2311 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.8528 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1658 | Regression_eta: 0.0196 | Regression_pt: 0.1554 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7178 | Classification_binary: 2.8528 | Regression_cos_phi: 0.5789 | Regression_energy: 1.8127 | Regression_eta: 0.8754 | Regression_pt: 1.8464 | Regression_sin_phi: 0.8910 | LR: 4.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3413s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.1078 | max_forward: 0.0170 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1224 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4875.3 | uss_MB: 4405.4 | child_rss_MB: 3201.0 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:38:51,239] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:02,127] INFO : Step 30000/50000 rank0 | Train Loss: 8.9942 | Train Loss Unweighted: 3.0947 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.7383 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1578 | Regression_eta: 0.0135 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7222 | Classification_binary: 2.7383 | Regression_cos_phi: 0.5116 | Regression_energy: 1.7252 | Regression_eta: 0.6036 | Regression_pt: 1.7610 | Regression_sin_phi: 0.9323 | LR: 4.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3900s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0807 | max_loss: 0.0035 | max_backward: 0.0332 | max_optimizer_step: 0.1230 | max_tensorboard_logging: 0.3752 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4875.3 | uss_MB: 4403.3 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:39:02,342] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:02,402] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 12:39:02,483] INFO : Running validation on rank0 +rank0 [2026-10-28 12:39:02,484] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:39:02,484] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:39:03,584] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 12:39:03,731] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:39:03,864] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:39:03,938] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:39:04,007] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:39:04,074] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:39:04,144] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:39:04,214] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:39:04,323] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:39:04,455] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:39:04,587] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:39:04,658] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:39:04,728] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:39:04,797] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:39:04,867] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:39:04,937] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:39:05,041] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:39:05,048] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:39:05,051] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:39:05,054] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:39:05,055] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:39:05,057] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:39:05,060] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:39:05,062] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:39:05,064] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:39:05,067] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:39:05,068] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:39:05,070] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:39:05,073] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:39:05,074] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:39:05,077] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:39:05,079] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:39:11,045] INFO : VALIDATION | Step=30000/50000 | Train Loss=8.9942 | Valid Loss=8.6269 | Train Loss Unweighted=3.0947 | Valid Loss Unweighted=3.1125 | Stale=1 | ETA=57.8m +rank0 [2026-10-28 12:39:11,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:11,109] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:39:11,112] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:39:11,278] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:12,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:13,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:14,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:14,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:14,926] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:39:14,930] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:14,986] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:39:14,989] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:15,099] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:16,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:17,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:17,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:18,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:18,439] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:39:18,443] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:18,495] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:39:18,498] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:39:18,602] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:20,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:20,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:21,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:21,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:21,797] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:39:21,805] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:39:22,065] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:39:22,474] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:23,921] INFO : Plotted jets +rank0 [2026-10-28 12:39:25,006] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:25,039] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:25,272] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:25,664] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:27,546] INFO : Plotted jets +rank0 [2026-10-28 12:39:28,644] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:28,676] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 12:39:28,886] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 12:39:29,249] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:30,630] INFO : Plotted jets +rank0 [2026-10-28 12:39:31,682] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:43,039] INFO : Step 30100/50000 rank0 | Train Loss: 8.4683 | Train Loss Unweighted: 3.1075 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.7584 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1541 | Regression_eta: 0.0105 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7982 | Classification_binary: 2.7584 | Regression_cos_phi: 0.4948 | Regression_energy: 1.6850 | Regression_eta: 0.4685 | Regression_pt: 1.7897 | Regression_sin_phi: 0.4736 | LR: 4.08e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4512s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1075 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.3957 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4944.1 | uss_MB: 4474.1 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:39:43,251] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:54,084] INFO : Step 30200/50000 rank0 | Train Loss: 8.5808 | Train Loss Unweighted: 2.9869 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6554 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1454 | Regression_eta: 0.0198 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7235 | Classification_binary: 2.6554 | Regression_cos_phi: 0.4684 | Regression_energy: 1.5895 | Regression_eta: 0.8845 | Regression_pt: 1.5748 | Regression_sin_phi: 0.6845 | LR: 4.06e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4457s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1249 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4944.1 | uss_MB: 4474.9 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:54,299] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:05,158] INFO : Step 30300/50000 rank0 | Train Loss: 8.8846 | Train Loss Unweighted: 3.2142 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.8558 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1585 | Regression_eta: 0.0167 | Regression_pt: 0.1507 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7280 | Classification_binary: 2.8558 | Regression_cos_phi: 0.4490 | Regression_energy: 1.7327 | Regression_eta: 0.7452 | Regression_pt: 1.7910 | Regression_sin_phi: 0.5829 | LR: 4.03e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4134s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1194 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1205 | max_tensorboard_logging: 0.3697 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4944.1 | uss_MB: 4475.3 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:05,372] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:16,255] INFO : Step 30400/50000 rank0 | Train Loss: 9.2181 | Train Loss Unweighted: 3.0798 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7084 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1624 | Regression_eta: 0.0238 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7928 | Classification_binary: 2.7084 | Regression_cos_phi: 0.4619 | Regression_energy: 1.7752 | Regression_eta: 1.0631 | Regression_pt: 1.7867 | Regression_sin_phi: 0.6299 | LR: 4.00e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.4029s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1107 | max_loss: 0.0033 | max_backward: 0.0342 | max_optimizer_step: 0.1228 | max_tensorboard_logging: 0.3619 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4944.1 | uss_MB: 4475.4 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:16,467] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:27,281] INFO : Step 30500/50000 rank0 | Train Loss: 9.0331 | Train Loss Unweighted: 3.0008 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6496 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1489 | Regression_eta: 0.0258 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7171 | Classification_binary: 2.6496 | Regression_cos_phi: 0.4712 | Regression_energy: 1.6277 | Regression_eta: 1.1507 | Regression_pt: 1.6940 | Regression_sin_phi: 0.7228 | LR: 3.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0147 | max_backward: 0.0339 | max_optimizer_step: 0.1235 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4944.1 | uss_MB: 4475.4 | child_rss_MB: 3197.8 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:27,492] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:38,496] INFO : Step 30600/50000 rank0 | Train Loss: 8.3329 | Train Loss Unweighted: 3.0949 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7635 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1450 | Regression_eta: 0.0110 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7659 | Classification_binary: 2.7635 | Regression_cos_phi: 0.4576 | Regression_energy: 1.5848 | Regression_eta: 0.4911 | Regression_pt: 1.6858 | Regression_sin_phi: 0.5842 | LR: 3.95e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3332s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0645 | max_forward: 0.0145 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1221 | max_tensorboard_logging: 0.3200 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4944.1 | uss_MB: 4475.5 | child_rss_MB: 3200.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:40:38,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:49,599] INFO : Step 30700/50000 rank0 | Train Loss: 9.2003 | Train Loss Unweighted: 3.1740 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.8052 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1646 | Regression_eta: 0.0209 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7397 | Classification_binary: 2.8052 | Regression_cos_phi: 0.4903 | Regression_energy: 1.7997 | Regression_eta: 0.9313 | Regression_pt: 1.7714 | Regression_sin_phi: 0.6629 | LR: 3.92e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3326s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0708 | max_forward: 0.0134 | max_loss: 0.0031 | max_backward: 0.0318 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.3120 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4942.9 | uss_MB: 4474.3 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:40:49,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:00,674] INFO : Step 30800/50000 rank0 | Train Loss: 7.9721 | Train Loss Unweighted: 3.0409 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.7356 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1340 | Regression_eta: 0.0108 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7053 | Classification_binary: 2.7356 | Regression_cos_phi: 0.4334 | Regression_energy: 1.4647 | Regression_eta: 0.4802 | Regression_pt: 1.5265 | Regression_sin_phi: 0.6264 | LR: 3.90e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3774s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0638 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3806 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4943.9 | uss_MB: 4475.1 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:41:00,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:11,794] INFO : Step 30900/50000 rank0 | Train Loss: 8.9041 | Train Loss Unweighted: 3.0738 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7171 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1587 | Regression_eta: 0.0165 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7646 | Classification_binary: 2.7171 | Regression_cos_phi: 0.3971 | Regression_energy: 1.7355 | Regression_eta: 0.7354 | Regression_pt: 1.7395 | Regression_sin_phi: 0.8149 | LR: 3.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4040s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0593 | max_loss: 0.0033 | max_backward: 0.0391 | max_optimizer_step: 0.1242 | max_tensorboard_logging: 0.3829 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4944.9 | uss_MB: 4477.8 | child_rss_MB: 3201.2 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:41:12,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:22,863] INFO : Step 31000/50000 rank0 | Train Loss: 8.7065 | Train Loss Unweighted: 3.0753 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.7166 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1575 | Regression_eta: 0.0111 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7718 | Classification_binary: 2.7166 | Regression_cos_phi: 0.4664 | Regression_energy: 1.7215 | Regression_eta: 0.4943 | Regression_pt: 1.8427 | Regression_sin_phi: 0.6932 | LR: 3.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3910s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0634 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4944.9 | uss_MB: 4475.8 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:41:23,078] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:23,114] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 12:41:23,197] INFO : Running validation on rank0 +rank0 [2026-10-28 12:41:23,197] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:41:23,198] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:41:24,278] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 12:41:24,424] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:41:24,556] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:41:24,630] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:41:24,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:41:24,766] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:41:24,836] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:41:24,905] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:41:25,014] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:41:25,148] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:41:25,279] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:41:25,350] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:41:25,420] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:41:25,488] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:41:25,558] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:41:25,629] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:41:25,733] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:41:25,739] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:41:25,742] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:41:25,744] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:41:25,746] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:41:25,749] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:41:25,750] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:41:25,753] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:41:25,756] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:41:25,759] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:41:25,761] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:41:25,762] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:41:25,765] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:41:25,768] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:41:25,770] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:41:25,771] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:41:31,623] INFO : VALIDATION | Step=31000/50000 | Train Loss=8.7065 | Valid Loss=8.5125 | Train Loss Unweighted=3.0753 | Valid Loss Unweighted=3.0729 | Stale=2 | ETA=54.5m +rank0 [2026-10-28 12:41:31,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:31,688] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:41:31,692] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:31,858] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:33,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:34,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:34,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:35,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:35,592] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:35,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:35,650] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:41:35,653] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:35,832] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:37,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:37,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:38,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:39,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:39,178] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:41:39,182] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:39,236] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:41:39,239] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:41:39,344] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:40,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:41,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:41,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:42,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:42,710] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:42,718] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:42,980] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:43,385] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:44,819] INFO : Plotted jets +rank0 [2026-10-28 12:41:45,900] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:45,933] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:46,163] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:46,556] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:48,428] INFO : Plotted jets +rank0 [2026-10-28 12:41:49,522] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:49,556] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:49,767] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:50,127] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:51,498] INFO : Plotted jets +rank0 [2026-10-28 12:41:52,542] INFO : Plotted jet ratio +rank0 [2026-10-28 12:42:03,955] INFO : Step 31100/50000 rank0 | Train Loss: 8.5782 | Train Loss Unweighted: 2.9896 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6411 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1548 | Regression_eta: 0.0166 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6968 | Classification_binary: 2.6411 | Regression_cos_phi: 0.4962 | Regression_energy: 1.6918 | Regression_eta: 0.7424 | Regression_pt: 1.7189 | Regression_sin_phi: 0.5911 | LR: 3.82e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3388s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.2175 | max_tensorboard_logging: 0.2925 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5020.9 | uss_MB: 4548.6 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:04,188] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:15,204] INFO : Step 31200/50000 rank0 | Train Loss: 8.3741 | Train Loss Unweighted: 3.0365 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6959 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1533 | Regression_eta: 0.0106 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7166 | Classification_binary: 2.6959 | Regression_cos_phi: 0.4515 | Regression_energy: 1.6755 | Regression_eta: 0.4724 | Regression_pt: 1.7098 | Regression_sin_phi: 0.6524 | LR: 3.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3568s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.1303 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5021.9 | uss_MB: 4549.6 | child_rss_MB: 3224.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:42:15,416] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:26,281] INFO : Step 31300/50000 rank0 | Train Loss: 8.4267 | Train Loss Unweighted: 3.0419 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6906 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1549 | Regression_eta: 0.0083 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8155 | Classification_binary: 2.6906 | Regression_cos_phi: 0.4539 | Regression_energy: 1.6931 | Regression_eta: 0.3680 | Regression_pt: 1.8219 | Regression_sin_phi: 0.5836 | LR: 3.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3783s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0168 | max_loss: 0.0033 | max_backward: 0.1463 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.2909 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5021.9 | uss_MB: 4552.2 | child_rss_MB: 3216.9 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:42:26,493] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:37,369] INFO : Step 31400/50000 rank0 | Train Loss: 9.0298 | Train Loss Unweighted: 3.0601 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6872 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1656 | Regression_eta: 0.0132 | Regression_pt: 0.1582 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7203 | Classification_binary: 2.6872 | Regression_cos_phi: 0.5304 | Regression_energy: 1.8106 | Regression_eta: 0.5876 | Regression_pt: 1.8793 | Regression_sin_phi: 0.8143 | LR: 3.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3562s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.1342 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5021.9 | uss_MB: 4550.5 | child_rss_MB: 3217.6 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:42:37,581] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:48,371] INFO : Step 31500/50000 rank0 | Train Loss: 8.8985 | Train Loss Unweighted: 3.1530 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.7992 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1534 | Regression_eta: 0.0134 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7383 | Classification_binary: 2.7992 | Regression_cos_phi: 0.4834 | Regression_energy: 1.6773 | Regression_eta: 0.5984 | Regression_pt: 1.7986 | Regression_sin_phi: 0.8033 | LR: 3.71e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0033 | max_backward: 0.0330 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5022.9 | uss_MB: 4550.5 | child_rss_MB: 3198.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:48,604] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:59,467] INFO : Step 31600/50000 rank0 | Train Loss: 8.4390 | Train Loss Unweighted: 3.0108 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.6645 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1536 | Regression_eta: 0.0103 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7120 | Classification_binary: 2.6645 | Regression_cos_phi: 0.4754 | Regression_energy: 1.6790 | Regression_eta: 0.4583 | Regression_pt: 1.7705 | Regression_sin_phi: 0.6793 | LR: 3.69e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3002s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5022.9 | uss_MB: 4550.7 | child_rss_MB: 3198.0 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:59,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:10,362] INFO : Step 31700/50000 rank0 | Train Loss: 9.2963 | Train Loss Unweighted: 3.2675 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.8981 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1603 | Regression_eta: 0.0148 | Regression_pt: 0.1566 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7982 | Classification_binary: 2.8981 | Regression_cos_phi: 0.5051 | Regression_energy: 1.7527 | Regression_eta: 0.6614 | Regression_pt: 1.8602 | Regression_sin_phi: 0.8206 | LR: 3.66e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3287s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0512 | max_forward: 0.0163 | max_loss: 0.0033 | max_backward: 0.0381 | max_optimizer_step: 0.1240 | max_tensorboard_logging: 0.3023 | max_tensorboard_flush: 0.0133 | Memory: rss_MB: 5022.2 | uss_MB: 4550.8 | child_rss_MB: 3199.9 | allocated_MB: 210.7 | reserved_MB: 2736.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:43:10,578] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:21,450] INFO : Step 31800/50000 rank0 | Train Loss: 7.9462 | Train Loss Unweighted: 2.9626 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.6545 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1331 | Regression_eta: 0.0090 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6807 | Classification_binary: 2.6545 | Regression_cos_phi: 0.4615 | Regression_energy: 1.4555 | Regression_eta: 0.4031 | Regression_pt: 1.5836 | Regression_sin_phi: 0.7073 | LR: 3.64e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4292s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0819 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5023.2 | uss_MB: 4549.9 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:43:21,662] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:32,443] INFO : Step 31900/50000 rank0 | Train Loss: 7.9949 | Train Loss Unweighted: 2.9342 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.6233 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1351 | Regression_eta: 0.0153 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6595 | Classification_binary: 2.6233 | Regression_cos_phi: 0.3513 | Regression_energy: 1.4768 | Regression_eta: 0.6829 | Regression_pt: 1.5524 | Regression_sin_phi: 0.6487 | LR: 3.61e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.1046 | max_backward: 0.0341 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5023.2 | uss_MB: 4550.1 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:43:32,654] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:43,537] INFO : Step 32000/50000 rank0 | Train Loss: 8.4876 | Train Loss Unweighted: 2.9761 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.6478 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1405 | Regression_eta: 0.0199 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6887 | Classification_binary: 2.6478 | Regression_cos_phi: 0.5081 | Regression_energy: 1.5356 | Regression_eta: 0.8874 | Regression_pt: 1.6090 | Regression_sin_phi: 0.6110 | LR: 3.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4162s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0561 | max_backward: 0.0334 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.3866 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5023.2 | uss_MB: 4550.5 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:43:43,749] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:43,768] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 12:43:43,953] INFO : Running validation on rank0 +rank0 [2026-10-28 12:43:43,954] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:43:43,954] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:43:45,033] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 12:43:45,179] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:43:45,311] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:43:45,386] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:43:45,456] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:43:45,523] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:43:45,593] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:43:45,662] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:43:45,771] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:43:45,903] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:43:46,035] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:43:46,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:43:46,178] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:43:46,247] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:43:46,317] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:43:46,387] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:43:46,497] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:43:46,503] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:43:46,507] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:43:46,509] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:43:46,510] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:43:46,513] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:43:46,516] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:43:46,519] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:43:46,520] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:43:46,523] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:43:46,525] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:43:46,528] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:43:46,529] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:43:46,532] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:43:46,534] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:43:46,537] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:43:52,401] INFO : VALIDATION | Step=32000/50000 | Train Loss=8.4876 | Valid Loss=8.3453 | Train Loss Unweighted=2.9761 | Valid Loss Unweighted=3.0147 | Stale=0 | ETA=51.4m +rank0 [2026-10-28 12:43:52,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:43:52,467] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:43:52,470] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:43:52,643] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:43:54,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:43:54,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:43:55,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:43:56,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:43:56,361] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:43:56,364] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:43:56,420] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:43:56,423] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:43:56,596] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:43:58,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:43:58,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:43:59,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:43:59,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:43:59,939] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:43:59,943] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:43:59,997] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:44:00,000] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:44:00,122] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:44:01,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:44:02,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:44:02,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:44:03,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:44:03,327] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:44:03,335] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:44:03,593] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:44:04,001] INFO : Plotted number of elements +rank0 [2026-10-28 12:44:05,474] INFO : Plotted jets +rank0 [2026-10-28 12:44:06,555] INFO : Plotted jet ratio +rank0 [2026-10-28 12:44:06,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:44:06,820] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:44:07,213] INFO : Plotted number of elements +rank0 [2026-10-28 12:44:08,611] INFO : Plotted jets +rank0 [2026-10-28 12:44:10,259] INFO : Plotted jet ratio +rank0 [2026-10-28 12:44:10,339] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 12:44:10,543] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 12:44:10,912] INFO : Plotted number of elements +rank0 [2026-10-28 12:44:12,301] INFO : Plotted jets +rank0 [2026-10-28 12:44:13,358] INFO : Plotted jet ratio +rank0 [2026-10-28 12:44:24,706] INFO : Step 32100/50000 rank0 | Train Loss: 8.5814 | Train Loss Unweighted: 3.0028 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6382 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1625 | Regression_eta: 0.0115 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7590 | Classification_binary: 2.6382 | Regression_cos_phi: 0.4249 | Regression_energy: 1.7769 | Regression_eta: 0.5131 | Regression_pt: 1.8724 | Regression_sin_phi: 0.5969 | LR: 3.56e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.4708s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0165 | max_loss: 0.0624 | max_backward: 0.0383 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3868 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5215.4 | uss_MB: 4747.7 | child_rss_MB: 3196.7 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:44:24,921] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:35,775] INFO : Step 32200/50000 rank0 | Train Loss: 8.9501 | Train Loss Unweighted: 3.1769 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8049 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1668 | Regression_eta: 0.0113 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7476 | Classification_binary: 2.8049 | Regression_cos_phi: 0.5090 | Regression_energy: 1.8234 | Regression_eta: 0.5054 | Regression_pt: 1.8906 | Regression_sin_phi: 0.6692 | LR: 3.53e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4158s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0547 | max_backward: 0.0338 | max_optimizer_step: 0.1275 | max_tensorboard_logging: 0.3808 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5096.2 | uss_MB: 4627.3 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:44:35,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:47,003] INFO : Step 32300/50000 rank0 | Train Loss: 8.2214 | Train Loss Unweighted: 2.9704 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6436 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1429 | Regression_eta: 0.0097 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7505 | Classification_binary: 2.6436 | Regression_cos_phi: 0.5937 | Regression_energy: 1.5628 | Regression_eta: 0.4338 | Regression_pt: 1.6499 | Regression_sin_phi: 0.5871 | LR: 3.51e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4173s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0671 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.3934 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 5097.2 | uss_MB: 4628.0 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:44:47,218] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:58,134] INFO : Step 32400/50000 rank0 | Train Loss: 8.3067 | Train Loss Unweighted: 2.9954 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6556 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1525 | Regression_eta: 0.0102 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6935 | Classification_binary: 2.6556 | Regression_cos_phi: 0.5204 | Regression_energy: 1.6668 | Regression_eta: 0.4566 | Regression_pt: 1.7145 | Regression_sin_phi: 0.5994 | LR: 3.48e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4073s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0476 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.4008 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5097.2 | uss_MB: 4628.1 | child_rss_MB: 3198.0 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:44:58,346] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:09,205] INFO : Step 32500/50000 rank0 | Train Loss: 8.3416 | Train Loss Unweighted: 2.9858 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.6567 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1447 | Regression_eta: 0.0131 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6621 | Classification_binary: 2.6567 | Regression_cos_phi: 0.5032 | Regression_energy: 1.5823 | Regression_eta: 0.5847 | Regression_pt: 1.6453 | Regression_sin_phi: 0.7073 | LR: 3.46e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3479s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0302 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1248 | max_tensorboard_logging: 0.3733 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5097.2 | uss_MB: 4628.1 | child_rss_MB: 3200.1 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:45:09,417] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:20,312] INFO : Step 32600/50000 rank0 | Train Loss: 8.8301 | Train Loss Unweighted: 3.1125 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7620 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1541 | Regression_eta: 0.0120 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8028 | Classification_binary: 2.7620 | Regression_cos_phi: 0.6391 | Regression_energy: 1.6843 | Regression_eta: 0.5349 | Regression_pt: 1.7374 | Regression_sin_phi: 0.6696 | LR: 3.43e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4426s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1110 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3710 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5097.2 | uss_MB: 4628.2 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:45:20,526] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:31,426] INFO : Step 32700/50000 rank0 | Train Loss: 8.4444 | Train Loss Unweighted: 3.0079 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6729 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1455 | Regression_eta: 0.0137 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7941 | Classification_binary: 2.6729 | Regression_cos_phi: 0.4349 | Regression_energy: 1.5902 | Regression_eta: 0.6118 | Regression_pt: 1.6749 | Regression_sin_phi: 0.6656 | LR: 3.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3669s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1051 | max_forward: 0.0136 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.2851 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5097.2 | uss_MB: 4628.5 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:45:31,639] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:42,555] INFO : Step 32800/50000 rank0 | Train Loss: 8.5610 | Train Loss Unweighted: 3.0593 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7064 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1555 | Regression_eta: 0.0126 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7337 | Classification_binary: 2.7064 | Regression_cos_phi: 0.4497 | Regression_energy: 1.7001 | Regression_eta: 0.5616 | Regression_pt: 1.8060 | Regression_sin_phi: 0.6035 | LR: 3.38e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3394s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0504 | max_forward: 0.0137 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5096.2 | uss_MB: 4627.6 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:45:42,769] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:53,506] INFO : Step 32900/50000 rank0 | Train Loss: 9.1049 | Train Loss Unweighted: 3.0897 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.7089 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1670 | Regression_eta: 0.0136 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7688 | Classification_binary: 2.7089 | Regression_cos_phi: 0.4743 | Regression_energy: 1.8259 | Regression_eta: 0.6073 | Regression_pt: 1.9533 | Regression_sin_phi: 0.7663 | LR: 3.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0797 | max_backward: 0.0332 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5096.2 | uss_MB: 4627.6 | child_rss_MB: 3199.6 | allocated_MB: 207.0 | reserved_MB: 2854.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:45:53,718] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:04,750] INFO : Step 33000/50000 rank0 | Train Loss: 7.8995 | Train Loss Unweighted: 2.9063 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5945 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1365 | Regression_eta: 0.0128 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6756 | Classification_binary: 2.5945 | Regression_cos_phi: 0.3831 | Regression_energy: 1.4925 | Regression_eta: 0.5690 | Regression_pt: 1.5710 | Regression_sin_phi: 0.6138 | LR: 3.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3854s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0327 | max_loss: 0.0034 | max_backward: 0.0385 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3802 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5096.2 | uss_MB: 4630.2 | child_rss_MB: 3200.5 | allocated_MB: 210.7 | reserved_MB: 2854.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:46:04,961] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:04,989] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 12:46:05,072] INFO : Running validation on rank0 +rank0 [2026-10-28 12:46:05,073] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:46:05,073] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:46:06,173] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 12:46:06,320] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:46:06,453] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:46:06,528] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:46:06,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:46:06,666] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:46:06,736] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:46:06,806] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:46:06,915] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:46:07,039] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:46:07,161] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:46:07,233] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:46:07,303] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:46:07,372] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:46:07,441] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:46:07,510] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:46:07,619] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:46:07,626] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:46:07,630] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:46:07,634] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:46:07,638] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:46:07,642] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:46:07,646] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:46:07,650] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:46:07,653] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:46:07,655] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:46:07,658] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:46:07,662] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:46:07,663] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:46:07,667] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:46:07,671] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:46:07,675] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:46:13,588] INFO : VALIDATION | Step=33000/50000 | Train Loss=7.8995 | Valid Loss=8.3631 | Train Loss Unweighted=2.9063 | Valid Loss Unweighted=3.0176 | Stale=1 | ETA=48.3m +rank0 [2026-10-28 12:46:13,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:13,657] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:46:13,660] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:46:13,829] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:15,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:16,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:16,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:17,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:17,519] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:46:17,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:17,578] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:46:17,581] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:17,704] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:19,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:19,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:20,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:20,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:21,117] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:46:21,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:21,174] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:46:21,177] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:46:21,290] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:22,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:23,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:23,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:24,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:24,564] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:46:24,572] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:46:24,836] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:46:25,242] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:26,731] INFO : Plotted jets +rank0 [2026-10-28 12:46:27,829] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:27,862] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:28,095] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:28,495] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:30,434] INFO : Plotted jets +rank0 [2026-10-28 12:46:31,538] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:31,571] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 12:46:31,782] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 12:46:32,144] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:33,523] INFO : Plotted jets +rank0 [2026-10-28 12:46:34,587] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:46,050] INFO : Step 33100/50000 rank0 | Train Loss: 8.4348 | Train Loss Unweighted: 3.0396 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7067 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1457 | Regression_eta: 0.0148 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7155 | Classification_binary: 2.7067 | Regression_cos_phi: 0.4622 | Regression_energy: 1.5931 | Regression_eta: 0.6597 | Regression_pt: 1.6573 | Regression_sin_phi: 0.6402 | LR: 3.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4993s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1195 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1256 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5293.5 | uss_MB: 4825.3 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:46:46,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:57,133] INFO : Step 33200/50000 rank0 | Train Loss: 7.9641 | Train Loss Unweighted: 2.9584 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6445 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1368 | Regression_eta: 0.0103 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.6445 | Regression_cos_phi: 0.4414 | Regression_energy: 1.4956 | Regression_eta: 0.4598 | Regression_pt: 1.6082 | Regression_sin_phi: 0.6446 | LR: 3.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3999s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0439 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1273 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5169.8 | uss_MB: 4700.3 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:46:57,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:08,251] INFO : Step 33300/50000 rank0 | Train Loss: 7.9828 | Train Loss Unweighted: 3.0623 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.7486 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1383 | Regression_eta: 0.0086 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7189 | Classification_binary: 2.7486 | Regression_cos_phi: 0.4388 | Regression_energy: 1.5115 | Regression_eta: 0.3852 | Regression_pt: 1.6026 | Regression_sin_phi: 0.5773 | LR: 3.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4362s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0514 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1261 | max_tensorboard_logging: 0.3926 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5169.8 | uss_MB: 4700.5 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:47:08,465] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:19,305] INFO : Step 33400/50000 rank0 | Train Loss: 8.7768 | Train Loss Unweighted: 3.0816 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7286 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1550 | Regression_eta: 0.0098 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7943 | Classification_binary: 2.7286 | Regression_cos_phi: 0.4634 | Regression_energy: 1.6942 | Regression_eta: 0.4358 | Regression_pt: 1.7930 | Regression_sin_phi: 0.8676 | LR: 3.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3387s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3764 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5170.8 | uss_MB: 4700.8 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:19,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:30,340] INFO : Step 33500/50000 rank0 | Train Loss: 8.5685 | Train Loss Unweighted: 2.9367 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5946 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1482 | Regression_eta: 0.0162 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6914 | Classification_binary: 2.5946 | Regression_cos_phi: 0.5011 | Regression_energy: 1.6199 | Regression_eta: 0.7219 | Regression_pt: 1.7110 | Regression_sin_phi: 0.7287 | LR: 3.21e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4096s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0333 | max_backward: 0.0334 | max_optimizer_step: 0.1262 | max_tensorboard_logging: 0.3922 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5170.8 | uss_MB: 4700.9 | child_rss_MB: 3196.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:30,552] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:41,573] INFO : Step 33600/50000 rank0 | Train Loss: 8.8185 | Train Loss Unweighted: 3.0817 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7129 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1654 | Regression_eta: 0.0139 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7447 | Classification_binary: 2.7129 | Regression_cos_phi: 0.4824 | Regression_energy: 1.8083 | Regression_eta: 0.6185 | Regression_pt: 1.8524 | Regression_sin_phi: 0.5993 | LR: 3.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4486s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0663 | max_loss: 0.0033 | max_backward: 0.0343 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.3990 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 5170.8 | uss_MB: 4701.1 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:47:41,785] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:52,633] INFO : Step 33700/50000 rank0 | Train Loss: 8.2483 | Train Loss Unweighted: 3.0850 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.7466 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1503 | Regression_eta: 0.0105 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6459 | Classification_binary: 2.7466 | Regression_cos_phi: 0.3985 | Regression_energy: 1.6433 | Regression_eta: 0.4682 | Regression_pt: 1.7596 | Regression_sin_phi: 0.5862 | LR: 3.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4612s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1091 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5171.8 | uss_MB: 4701.3 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:52,848] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:03,736] INFO : Step 33800/50000 rank0 | Train Loss: 8.5199 | Train Loss Unweighted: 3.0325 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6685 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1628 | Regression_eta: 0.0100 | Regression_pt: 0.1589 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7425 | Classification_binary: 2.6685 | Regression_cos_phi: 0.4161 | Regression_energy: 1.7800 | Regression_eta: 0.4446 | Regression_pt: 1.8885 | Regression_sin_phi: 0.5797 | LR: 3.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3961s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0333 | max_loss: 0.0036 | max_backward: 0.0349 | max_optimizer_step: 0.1294 | max_tensorboard_logging: 0.3627 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5170.5 | uss_MB: 4700.4 | child_rss_MB: 3201.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:48:03,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:14,824] INFO : Step 33900/50000 rank0 | Train Loss: 8.6038 | Train Loss Unweighted: 2.9823 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6427 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1476 | Regression_eta: 0.0188 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6926 | Classification_binary: 2.6427 | Regression_cos_phi: 0.4657 | Regression_energy: 1.6132 | Regression_eta: 0.8364 | Regression_pt: 1.6701 | Regression_sin_phi: 0.6831 | LR: 3.11e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0584 | max_backward: 0.0340 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.3969 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5170.5 | uss_MB: 4700.3 | child_rss_MB: 3201.8 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:48:15,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:26,143] INFO : Step 34000/50000 rank0 | Train Loss: 8.7485 | Train Loss Unweighted: 2.9757 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.6261 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1533 | Regression_eta: 0.0149 | Regression_pt: 0.1441 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7908 | Classification_binary: 2.6261 | Regression_cos_phi: 0.5675 | Regression_energy: 1.6765 | Regression_eta: 0.6667 | Regression_pt: 1.7127 | Regression_sin_phi: 0.7084 | LR: 3.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4657s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1141 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5170.5 | uss_MB: 4700.8 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:48:26,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:26,390] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 12:48:26,473] INFO : Running validation on rank0 +rank0 [2026-10-28 12:48:26,473] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:48:26,473] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:48:27,572] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 12:48:27,718] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:48:27,851] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:48:27,925] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:48:27,996] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:48:28,064] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:48:28,134] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:48:28,204] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:48:28,312] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:48:28,437] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:48:28,560] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:48:28,633] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:48:28,703] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:48:28,772] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:48:28,842] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:48:28,912] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:48:29,022] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:48:29,029] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:48:29,032] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:48:29,034] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:48:29,035] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:48:29,038] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:48:29,041] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:48:29,044] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:48:29,046] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:48:29,049] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:48:29,052] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:48:29,054] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:48:29,055] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:48:29,058] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:48:29,060] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:48:29,061] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:48:35,141] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.7485 | Valid Loss=8.2430 | Train Loss Unweighted=2.9757 | Valid Loss Unweighted=2.9711 | Stale=0 | ETA=45.2m +rank0 [2026-10-28 12:48:35,142] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:48:35,199] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:48:35,204] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:48:35,379] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:48:36,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:48:37,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:48:37,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:48:38,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:48:38,690] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:48:38,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:48:38,748] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:48:38,751] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:48:38,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:48:40,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:48:41,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:48:41,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:48:42,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:48:42,341] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:48:42,345] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:48:42,399] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:48:42,402] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:48:42,516] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:48:44,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:48:44,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:48:45,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:48:45,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:48:45,732] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:48:45,739] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:48:46,006] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:48:46,416] INFO : Plotted number of elements +rank0 [2026-10-28 12:48:47,867] INFO : Plotted jets +rank0 [2026-10-28 12:48:48,956] INFO : Plotted jet ratio +rank0 [2026-10-28 12:48:48,990] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:48:49,219] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:48:49,614] INFO : Plotted number of elements +rank0 [2026-10-28 12:48:51,007] INFO : Plotted jets +rank0 [2026-10-28 12:48:52,623] INFO : Plotted jet ratio +rank0 [2026-10-28 12:48:52,654] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 12:48:52,861] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 12:48:53,230] INFO : Plotted number of elements +rank0 [2026-10-28 12:48:54,616] INFO : Plotted jets +rank0 [2026-10-28 12:48:55,738] INFO : Plotted jet ratio +rank0 [2026-10-28 12:49:07,151] INFO : Step 34100/50000 rank0 | Train Loss: 8.3385 | Train Loss Unweighted: 3.0091 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.6780 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1452 | Regression_eta: 0.0118 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7389 | Classification_binary: 2.6780 | Regression_cos_phi: 0.4757 | Regression_energy: 1.5875 | Regression_eta: 0.5265 | Regression_pt: 1.6654 | Regression_sin_phi: 0.6667 | LR: 3.06e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4008s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0223 | max_loss: 0.0033 | max_backward: 0.0348 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.3931 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5363.9 | uss_MB: 4894.4 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:49:07,364] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:18,258] INFO : Step 34200/50000 rank0 | Train Loss: 8.8190 | Train Loss Unweighted: 3.1324 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7574 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1669 | Regression_eta: 0.0088 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6972 | Classification_binary: 2.7574 | Regression_cos_phi: 0.4431 | Regression_energy: 1.8252 | Regression_eta: 0.3927 | Regression_pt: 1.9756 | Regression_sin_phi: 0.7279 | LR: 3.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4084s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1068 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3142 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5241.8 | uss_MB: 4773.3 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:18,474] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:29,358] INFO : Step 34300/50000 rank0 | Train Loss: 8.1188 | Train Loss Unweighted: 2.9349 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6307 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1300 | Regression_eta: 0.0138 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7211 | Classification_binary: 2.6307 | Regression_cos_phi: 0.5174 | Regression_energy: 1.4209 | Regression_eta: 0.6175 | Regression_pt: 1.4933 | Regression_sin_phi: 0.7180 | LR: 3.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3908s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0897 | max_forward: 0.0180 | max_loss: 0.0035 | max_backward: 0.0393 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.2953 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5242.8 | uss_MB: 4776.4 | child_rss_MB: 3195.2 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:49:29,570] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:40,623] INFO : Step 34400/50000 rank0 | Train Loss: 8.6032 | Train Loss Unweighted: 3.0206 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6695 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1554 | Regression_eta: 0.0142 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6789 | Classification_binary: 2.6695 | Regression_cos_phi: 0.4902 | Regression_energy: 1.6984 | Regression_eta: 0.6347 | Regression_pt: 1.7700 | Regression_sin_phi: 0.6615 | LR: 2.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3468s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.2688 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5242.8 | uss_MB: 4774.9 | child_rss_MB: 3192.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:40,847] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:51,724] INFO : Step 34500/50000 rank0 | Train Loss: 7.5407 | Train Loss Unweighted: 2.9423 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.6333 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1386 | Regression_eta: 0.0077 | Regression_pt: 0.1352 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6682 | Classification_binary: 2.6333 | Regression_cos_phi: 0.3403 | Regression_energy: 1.5153 | Regression_eta: 0.3436 | Regression_pt: 1.6062 | Regression_sin_phi: 0.4339 | LR: 2.97e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3368s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0130 | Memory: rss_MB: 5242.8 | uss_MB: 4775.4 | child_rss_MB: 3194.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:51,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:02,834] INFO : Step 34600/50000 rank0 | Train Loss: 8.7762 | Train Loss Unweighted: 3.0292 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6894 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1475 | Regression_eta: 0.0174 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8151 | Classification_binary: 2.6894 | Regression_cos_phi: 0.4474 | Regression_energy: 1.6129 | Regression_eta: 0.7770 | Regression_pt: 1.6380 | Regression_sin_phi: 0.7964 | LR: 2.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.2958 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5242.8 | uss_MB: 4775.4 | child_rss_MB: 3193.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:50:03,049] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:13,670] INFO : Step 34700/50000 rank0 | Train Loss: 8.1281 | Train Loss Unweighted: 3.0442 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7223 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1411 | Regression_eta: 0.0077 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6950 | Classification_binary: 2.7223 | Regression_cos_phi: 0.5339 | Regression_energy: 1.5429 | Regression_eta: 0.3425 | Regression_pt: 1.6609 | Regression_sin_phi: 0.6306 | LR: 2.92e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3106s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0571 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.2774 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5242.8 | uss_MB: 4775.4 | child_rss_MB: 3193.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:50:13,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:24,791] INFO : Step 34800/50000 rank0 | Train Loss: 8.1722 | Train Loss Unweighted: 2.8989 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5651 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1449 | Regression_eta: 0.0092 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7600 | Classification_binary: 2.5651 | Regression_cos_phi: 0.5810 | Regression_energy: 1.5837 | Regression_eta: 0.4100 | Regression_pt: 1.7197 | Regression_sin_phi: 0.5525 | LR: 2.90e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3630s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0725 | max_forward: 0.0173 | max_loss: 0.0035 | max_backward: 0.0382 | max_optimizer_step: 0.1275 | max_tensorboard_logging: 0.2998 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5241.1 | uss_MB: 4775.5 | child_rss_MB: 3201.2 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 12:50:25,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:35,912] INFO : Step 34900/50000 rank0 | Train Loss: 9.0481 | Train Loss Unweighted: 3.0615 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.6655 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1771 | Regression_eta: 0.0127 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7018 | Classification_binary: 2.6655 | Regression_cos_phi: 0.4905 | Regression_energy: 1.9363 | Regression_eta: 0.5675 | Regression_pt: 2.0595 | Regression_sin_phi: 0.6271 | LR: 2.88e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4051s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0749 | max_forward: 0.0135 | max_loss: 0.0035 | max_backward: 0.0330 | max_optimizer_step: 0.1268 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 5241.1 | uss_MB: 4774.4 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:50:36,128] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:46,972] INFO : Step 35000/50000 rank0 | Train Loss: 8.2289 | Train Loss Unweighted: 2.9858 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.6656 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1404 | Regression_eta: 0.0110 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7033 | Classification_binary: 2.6656 | Regression_cos_phi: 0.5227 | Regression_energy: 1.5351 | Regression_eta: 0.4909 | Regression_pt: 1.5977 | Regression_sin_phi: 0.7136 | LR: 2.85e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4003s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1015 | max_forward: 0.0170 | max_loss: 0.0033 | max_backward: 0.0389 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.2963 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5242.1 | uss_MB: 4777.1 | child_rss_MB: 3198.7 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:50:47,188] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:47,206] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 12:50:47,290] INFO : Running validation on rank0 +rank0 [2026-10-28 12:50:47,290] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:50:47,291] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:50:48,397] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 12:50:48,546] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:50:48,680] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:50:48,754] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:50:48,823] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:50:48,891] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:50:48,961] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:50:49,030] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:50:49,140] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:50:49,273] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:50:49,406] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:50:49,477] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:50:49,547] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:50:49,616] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:50:49,687] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:50:49,757] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:50:49,868] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:50:49,874] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:50:49,877] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:50:49,880] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:50:49,883] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:50:49,884] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:50:49,888] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:50:49,890] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:50:49,891] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:50:49,894] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:50:50,000] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:50:50,003] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:50:50,005] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:50:50,006] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:50:50,009] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:50:50,012] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:50:55,957] INFO : VALIDATION | Step=35000/50000 | Train Loss=8.2289 | Valid Loss=8.3163 | Train Loss Unweighted=2.9858 | Valid Loss Unweighted=2.9814 | Stale=1 | ETA=42.2m +rank0 [2026-10-28 12:50:55,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:50:56,021] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:50:56,025] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:50:56,195] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:50:57,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:50:58,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:50:58,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:50:59,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:50:59,732] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:50:59,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:50:59,789] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:50:59,792] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:50:59,904] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:51:01,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:51:01,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:51:02,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:51:03,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:51:03,231] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:51:03,234] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:51:03,287] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:51:03,290] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:51:03,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:51:04,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:51:05,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:51:05,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:51:06,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:51:06,735] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:51:06,743] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:51:07,004] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:51:07,410] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:08,859] INFO : Plotted jets +rank0 [2026-10-28 12:51:09,958] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:09,991] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:51:10,226] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:51:10,622] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:12,015] INFO : Plotted jets +rank0 [2026-10-28 12:51:13,670] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:13,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 12:51:13,913] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 12:51:14,279] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:15,661] INFO : Plotted jets +rank0 [2026-10-28 12:51:16,725] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:28,045] INFO : Step 35100/50000 rank0 | Train Loss: 8.3284 | Train Loss Unweighted: 2.8907 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5623 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1406 | Regression_eta: 0.0153 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7815 | Classification_binary: 2.5623 | Regression_cos_phi: 0.5260 | Regression_energy: 1.5370 | Regression_eta: 0.6844 | Regression_pt: 1.6306 | Regression_sin_phi: 0.6065 | LR: 2.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4442s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0159 | max_loss: 0.0545 | max_backward: 0.0382 | max_optimizer_step: 0.1275 | max_tensorboard_logging: 0.4023 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 5428.8 | uss_MB: 4962.6 | child_rss_MB: 3197.8 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:51:28,259] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:39,165] INFO : Step 35200/50000 rank0 | Train Loss: 7.7358 | Train Loss Unweighted: 2.9265 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.6100 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1426 | Regression_eta: 0.0099 | Regression_pt: 0.1358 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6572 | Classification_binary: 2.6100 | Regression_cos_phi: 0.3594 | Regression_energy: 1.5589 | Regression_eta: 0.4408 | Regression_pt: 1.6138 | Regression_sin_phi: 0.4958 | LR: 2.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4828s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0702 | max_backward: 0.0332 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.4188 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5317.7 | uss_MB: 4848.5 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:51:39,378] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:50,373] INFO : Step 35300/50000 rank0 | Train Loss: 8.1360 | Train Loss Unweighted: 2.9387 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6074 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1460 | Regression_eta: 0.0097 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6776 | Classification_binary: 2.6074 | Regression_cos_phi: 0.4708 | Regression_energy: 1.5965 | Regression_eta: 0.4308 | Regression_pt: 1.7055 | Regression_sin_phi: 0.6475 | LR: 2.79e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.4164s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0243 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3992 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5318.7 | uss_MB: 4849.6 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:51:50,587] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:01,439] INFO : Step 35400/50000 rank0 | Train Loss: 7.8836 | Train Loss Unweighted: 2.9146 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6095 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1329 | Regression_eta: 0.0092 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7844 | Classification_binary: 2.6095 | Regression_cos_phi: 0.4326 | Regression_energy: 1.4529 | Regression_eta: 0.4090 | Regression_pt: 1.5256 | Regression_sin_phi: 0.6696 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4874s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1118 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1297 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5318.7 | uss_MB: 4849.8 | child_rss_MB: 3194.7 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:52:01,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:12,518] INFO : Step 35500/50000 rank0 | Train Loss: 8.5601 | Train Loss Unweighted: 3.0585 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7013 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1618 | Regression_eta: 0.0103 | Regression_pt: 0.1522 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7156 | Classification_binary: 2.7013 | Regression_cos_phi: 0.4536 | Regression_energy: 1.7694 | Regression_eta: 0.4588 | Regression_pt: 1.8086 | Regression_sin_phi: 0.6529 | LR: 2.74e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4075s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0633 | max_loss: 0.0034 | max_backward: 0.0388 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3705 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5319.7 | uss_MB: 4852.3 | child_rss_MB: 3193.4 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:52:12,730] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:23,658] INFO : Step 35600/50000 rank0 | Train Loss: 8.4640 | Train Loss Unweighted: 2.9692 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6124 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1582 | Regression_eta: 0.0113 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7311 | Classification_binary: 2.6124 | Regression_cos_phi: 0.3885 | Regression_energy: 1.7290 | Regression_eta: 0.5020 | Regression_pt: 1.8417 | Regression_sin_phi: 0.6592 | LR: 2.72e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4066s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0369 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.3834 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5319.7 | uss_MB: 4850.2 | child_rss_MB: 3196.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:52:23,869] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:34,647] INFO : Step 35700/50000 rank0 | Train Loss: 8.2471 | Train Loss Unweighted: 3.0628 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7434 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1405 | Regression_eta: 0.0076 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7120 | Classification_binary: 2.7434 | Regression_cos_phi: 0.5157 | Regression_energy: 1.5357 | Regression_eta: 0.3375 | Regression_pt: 1.6186 | Regression_sin_phi: 0.7842 | LR: 2.70e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4826s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0133 | max_loss: 0.0827 | max_backward: 0.0330 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5319.7 | uss_MB: 4850.2 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:52:34,857] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:45,851] INFO : Step 35800/50000 rank0 | Train Loss: 8.2720 | Train Loss Unweighted: 2.9460 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5977 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1560 | Regression_eta: 0.0089 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7133 | Classification_binary: 2.5977 | Regression_cos_phi: 0.4603 | Regression_energy: 1.7059 | Regression_eta: 0.3981 | Regression_pt: 1.7932 | Regression_sin_phi: 0.6034 | LR: 2.68e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4909s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1231 | max_loss: 0.0035 | max_backward: 0.0387 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3771 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 5319.0 | uss_MB: 4850.7 | child_rss_MB: 3197.7 | allocated_MB: 210.7 | reserved_MB: 2730.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:52:46,065] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:56,806] INFO : Step 35900/50000 rank0 | Train Loss: 7.9205 | Train Loss Unweighted: 2.9577 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6349 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1433 | Regression_eta: 0.0077 | Regression_pt: 0.1401 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7573 | Classification_binary: 2.6349 | Regression_cos_phi: 0.3218 | Regression_energy: 1.5663 | Regression_eta: 0.3446 | Regression_pt: 1.6643 | Regression_sin_phi: 0.6313 | LR: 2.65e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4117s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0628 | max_backward: 0.0332 | max_optimizer_step: 0.1280 | max_tensorboard_logging: 0.3707 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5319.0 | uss_MB: 4849.5 | child_rss_MB: 3195.2 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:52:57,019] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:07,702] INFO : Step 36000/50000 rank0 | Train Loss: 9.1170 | Train Loss Unweighted: 3.1454 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7841 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1610 | Regression_eta: 0.0228 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7169 | Classification_binary: 2.7841 | Regression_cos_phi: 0.5516 | Regression_energy: 1.7597 | Regression_eta: 1.0162 | Regression_pt: 1.7099 | Regression_sin_phi: 0.5787 | LR: 2.63e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2859s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0673 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5319.0 | uss_MB: 4849.7 | child_rss_MB: 3220.1 | allocated_MB: 207.0 | reserved_MB: 2730.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:53:07,917] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:07,937] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 12:53:08,021] INFO : Running validation on rank0 +rank0 [2026-10-28 12:53:08,021] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:53:08,021] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:53:09,210] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 12:53:09,355] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:53:09,487] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:53:09,562] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:53:09,631] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:53:09,700] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:53:09,771] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:53:09,842] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:53:09,953] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:53:10,087] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:53:10,219] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:53:10,291] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:53:10,361] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:53:10,430] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:53:10,502] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:53:10,573] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:53:10,682] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:53:10,689] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:53:10,691] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:53:10,692] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:53:10,696] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:53:10,698] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:53:10,699] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:53:10,702] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:53:10,705] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:53:10,708] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:53:10,709] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:53:10,711] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:53:10,712] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:53:10,714] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:53:10,717] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:53:10,718] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:53:16,578] INFO : VALIDATION | Step=36000/50000 | Train Loss=9.1170 | Valid Loss=8.2770 | Train Loss Unweighted=3.1454 | Valid Loss Unweighted=2.9803 | Stale=2 | ETA=39.2m +rank0 [2026-10-28 12:53:16,586] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:16,644] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:53:16,648] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:53:16,868] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:18,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:19,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:19,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:20,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:20,514] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:53:20,518] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:20,573] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:53:20,576] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:20,693] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:22,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:22,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:23,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:23,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:23,957] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:53:23,960] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:24,013] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:53:24,016] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:53:24,131] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:25,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:26,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:26,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:27,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:27,582] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:53:27,590] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:53:27,848] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:53:28,267] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:29,706] INFO : Plotted jets +rank0 [2026-10-28 12:53:30,800] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:30,833] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:31,066] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:31,460] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:32,858] INFO : Plotted jets +rank0 [2026-10-28 12:53:34,469] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:34,502] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 12:53:34,712] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 12:53:35,077] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:36,453] INFO : Plotted jets +rank0 [2026-10-28 12:53:37,512] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:48,856] INFO : Step 36100/50000 rank0 | Train Loss: 7.8103 | Train Loss Unweighted: 2.9499 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6354 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1406 | Regression_eta: 0.0098 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7290 | Classification_binary: 2.6354 | Regression_cos_phi: 0.3428 | Regression_energy: 1.5367 | Regression_eta: 0.4380 | Regression_pt: 1.5889 | Regression_sin_phi: 0.5396 | LR: 2.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4449s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1317 | max_tensorboard_logging: 0.3981 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5513.3 | uss_MB: 5043.9 | child_rss_MB: 3193.9 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:49,074] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:53:59,952] INFO : Step 36200/50000 rank0 | Train Loss: 7.8587 | Train Loss Unweighted: 2.8574 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5409 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1388 | Regression_eta: 0.0077 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7148 | Classification_binary: 2.5409 | Regression_cos_phi: 0.3797 | Regression_energy: 1.5169 | Regression_eta: 0.3428 | Regression_pt: 1.6335 | Regression_sin_phi: 0.7300 | LR: 2.59e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4947s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0134 | max_loss: 0.0752 | max_backward: 0.0328 | max_optimizer_step: 0.1285 | max_tensorboard_logging: 0.3954 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 5513.3 | uss_MB: 5043.9 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:54:00,165] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:11,146] INFO : Step 36300/50000 rank0 | Train Loss: 8.5631 | Train Loss Unweighted: 3.0412 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6711 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1677 | Regression_eta: 0.0078 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6914 | Classification_binary: 2.6711 | Regression_cos_phi: 0.4340 | Regression_energy: 1.8336 | Regression_eta: 0.3488 | Regression_pt: 1.9329 | Regression_sin_phi: 0.6515 | LR: 2.57e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4081s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0234 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3998 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 5394.7 | uss_MB: 4924.8 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:54:11,364] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:22,202] INFO : Step 36400/50000 rank0 | Train Loss: 7.5979 | Train Loss Unweighted: 2.8414 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5524 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1250 | Regression_eta: 0.0090 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7531 | Classification_binary: 2.5524 | Regression_cos_phi: 0.4524 | Regression_energy: 1.3670 | Regression_eta: 0.3999 | Regression_pt: 1.4422 | Regression_sin_phi: 0.6308 | LR: 2.55e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3442s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0844 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.2863 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5395.7 | uss_MB: 4925.0 | child_rss_MB: 3194.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:54:22,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:33,303] INFO : Step 36500/50000 rank0 | Train Loss: 8.3497 | Train Loss Unweighted: 2.9112 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5655 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1532 | Regression_eta: 0.0112 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7607 | Classification_binary: 2.5655 | Regression_cos_phi: 0.4211 | Regression_energy: 1.6752 | Regression_eta: 0.4994 | Regression_pt: 1.7516 | Regression_sin_phi: 0.6762 | LR: 2.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3811s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0173 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5396.7 | uss_MB: 4927.6 | child_rss_MB: 3192.4 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:54:33,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:44,408] INFO : Step 36600/50000 rank0 | Train Loss: 9.0300 | Train Loss Unweighted: 3.1079 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7445 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1602 | Regression_eta: 0.0155 | Regression_pt: 0.1513 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7455 | Classification_binary: 2.7445 | Regression_cos_phi: 0.5862 | Regression_energy: 1.7514 | Regression_eta: 0.6904 | Regression_pt: 1.7983 | Regression_sin_phi: 0.7136 | LR: 2.50e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4443s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0845 | max_loss: 0.0035 | max_backward: 0.0341 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3619 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5396.7 | uss_MB: 4925.5 | child_rss_MB: 3192.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:54:44,620] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:55,517] INFO : Step 36700/50000 rank0 | Train Loss: 8.8857 | Train Loss Unweighted: 3.1248 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7694 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1568 | Regression_eta: 0.0161 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7276 | Classification_binary: 2.7694 | Regression_cos_phi: 0.4736 | Regression_energy: 1.7147 | Regression_eta: 0.7184 | Regression_pt: 1.7618 | Regression_sin_phi: 0.7203 | LR: 2.48e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0605 | max_loss: 0.0036 | max_backward: 0.0333 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5396.7 | uss_MB: 4925.6 | child_rss_MB: 3192.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:54:55,729] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:06,630] INFO : Step 36800/50000 rank0 | Train Loss: 8.4058 | Train Loss Unweighted: 3.0876 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7543 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1468 | Regression_eta: 0.0096 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7157 | Classification_binary: 2.7543 | Regression_cos_phi: 0.5575 | Regression_energy: 1.6054 | Regression_eta: 0.4272 | Regression_pt: 1.6904 | Regression_sin_phi: 0.6553 | LR: 2.46e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3838s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0887 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0384 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5396.7 | uss_MB: 4927.6 | child_rss_MB: 3194.8 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:55:06,846] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:17,705] INFO : Step 36900/50000 rank0 | Train Loss: 8.2571 | Train Loss Unweighted: 2.9882 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6650 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1408 | Regression_eta: 0.0123 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7247 | Classification_binary: 2.6650 | Regression_cos_phi: 0.4916 | Regression_energy: 1.5395 | Regression_eta: 0.5486 | Regression_pt: 1.6182 | Regression_sin_phi: 0.6694 | LR: 2.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3913s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1021 | max_forward: 0.0155 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.2894 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5395.1 | uss_MB: 4924.9 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:55:17,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:28,785] INFO : Step 37000/50000 rank0 | Train Loss: 8.5764 | Train Loss Unweighted: 2.9920 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6459 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1530 | Regression_eta: 0.0139 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6730 | Classification_binary: 2.6459 | Regression_cos_phi: 0.4610 | Regression_energy: 1.6731 | Regression_eta: 0.6216 | Regression_pt: 1.7346 | Regression_sin_phi: 0.7671 | LR: 2.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0201 | max_forward: 0.0168 | max_loss: 0.0035 | max_backward: 0.0383 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5396.1 | uss_MB: 4927.4 | child_rss_MB: 3196.1 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:55:28,996] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:29,013] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 12:55:29,097] INFO : Running validation on rank0 +rank0 [2026-10-28 12:55:29,097] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:55:29,097] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:55:30,187] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 12:55:30,335] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:55:30,469] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:55:30,543] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:55:30,612] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:55:30,680] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:55:30,750] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:55:30,820] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:55:30,931] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:55:31,064] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:55:31,197] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:55:31,270] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:55:31,339] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:55:31,408] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:55:31,477] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:55:31,548] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:55:31,660] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:55:31,667] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:55:31,670] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:55:31,673] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:55:31,673] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:55:31,674] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:55:31,677] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:55:31,680] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:55:31,682] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:55:31,684] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:55:31,686] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:55:31,687] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:55:31,690] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:55:31,692] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:55:31,694] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:55:31,696] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:55:37,569] INFO : VALIDATION | Step=37000/50000 | Train Loss=8.5764 | Valid Loss=8.1542 | Train Loss Unweighted=2.9920 | Valid Loss Unweighted=2.9461 | Stale=0 | ETA=36.2m +rank0 [2026-10-28 12:55:37,578] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:55:37,635] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:55:37,639] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:55:37,811] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:55:39,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:55:40,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:55:40,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:55:41,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:55:41,509] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:55:41,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:55:41,566] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:55:41,577] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:55:41,757] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:55:43,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:55:43,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:55:44,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:55:45,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:55:45,181] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:55:45,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:55:45,239] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:55:45,242] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:55:45,352] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:55:46,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:55:47,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:55:47,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:55:48,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:55:48,641] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:55:48,649] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:55:48,911] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:55:49,318] INFO : Plotted number of elements +rank0 [2026-10-28 12:55:50,767] INFO : Plotted jets +rank0 [2026-10-28 12:55:51,858] INFO : Plotted jet ratio +rank0 [2026-10-28 12:55:51,890] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:55:52,123] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:55:52,516] INFO : Plotted number of elements +rank0 [2026-10-28 12:55:54,416] INFO : Plotted jets +rank0 [2026-10-28 12:55:55,518] INFO : Plotted jet ratio +rank0 [2026-10-28 12:55:55,551] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 12:55:55,766] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 12:55:56,130] INFO : Plotted number of elements +rank0 [2026-10-28 12:55:57,513] INFO : Plotted jets +rank0 [2026-10-28 12:55:58,576] INFO : Plotted jet ratio +rank0 [2026-10-28 12:56:09,574] INFO : Step 37100/50000 rank0 | Train Loss: 8.0368 | Train Loss Unweighted: 3.0025 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6859 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1389 | Regression_eta: 0.0080 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7136 | Classification_binary: 2.6859 | Regression_cos_phi: 0.5275 | Regression_energy: 1.5181 | Regression_eta: 0.3581 | Regression_pt: 1.6172 | Regression_sin_phi: 0.6163 | LR: 2.40e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3213s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5583.5 | uss_MB: 5112.8 | child_rss_MB: 3197.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:09,788] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:20,447] INFO : Step 37200/50000 rank0 | Train Loss: 8.0984 | Train Loss Unweighted: 2.8799 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.5690 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1337 | Regression_eta: 0.0149 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8366 | Classification_binary: 2.5690 | Regression_cos_phi: 0.4368 | Regression_energy: 1.4620 | Regression_eta: 0.6660 | Regression_pt: 1.5060 | Regression_sin_phi: 0.6220 | LR: 2.38e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0216 | max_forward: 0.0147 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5583.5 | uss_MB: 5112.8 | child_rss_MB: 3193.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:56:20,659] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 100.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:31,492] INFO : Step 37300/50000 rank0 | Train Loss: 7.7696 | Train Loss Unweighted: 2.9498 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6368 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1390 | Regression_eta: 0.0094 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6913 | Classification_binary: 2.6368 | Regression_cos_phi: 0.3606 | Regression_energy: 1.5196 | Regression_eta: 0.4203 | Regression_pt: 1.6040 | Regression_sin_phi: 0.5368 | LR: 2.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3273s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0798 | max_forward: 0.0144 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5468.9 | uss_MB: 4997.6 | child_rss_MB: 3194.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:31,705] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:42,584] INFO : Step 37400/50000 rank0 | Train Loss: 8.3098 | Train Loss Unweighted: 2.8793 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5563 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1391 | Regression_eta: 0.0273 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6404 | Classification_binary: 2.5563 | Regression_cos_phi: 0.3426 | Regression_energy: 1.5212 | Regression_eta: 1.2185 | Regression_pt: 1.5315 | Regression_sin_phi: 0.4994 | LR: 2.34e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4214s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0862 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1301 | max_tensorboard_logging: 0.3571 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5468.9 | uss_MB: 4997.8 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:42,799] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:53,684] INFO : Step 37500/50000 rank0 | Train Loss: 8.7786 | Train Loss Unweighted: 3.0007 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6424 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1581 | Regression_eta: 0.0114 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7657 | Classification_binary: 2.6424 | Regression_cos_phi: 0.5314 | Regression_energy: 1.7290 | Regression_eta: 0.5106 | Regression_pt: 1.8030 | Regression_sin_phi: 0.7964 | LR: 2.32e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3827s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0337 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:53,895] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:04,769] INFO : Step 37600/50000 rank0 | Train Loss: 7.8635 | Train Loss Unweighted: 2.8732 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5549 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1400 | Regression_eta: 0.0080 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7033 | Classification_binary: 2.5549 | Regression_cos_phi: 0.5197 | Regression_energy: 1.5305 | Regression_eta: 0.3569 | Regression_pt: 1.6358 | Regression_sin_phi: 0.5623 | LR: 2.30e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3877s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0501 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3592 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:57:04,982] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:15,882] INFO : Step 37700/50000 rank0 | Train Loss: 8.1664 | Train Loss Unweighted: 2.8295 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4849 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1535 | Regression_eta: 0.0090 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6949 | Classification_binary: 2.4849 | Regression_cos_phi: 0.4767 | Regression_energy: 1.6786 | Regression_eta: 0.4017 | Regression_pt: 1.7748 | Regression_sin_phi: 0.6549 | LR: 2.28e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3743s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1067 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1290 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:57:16,095] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:26,994] INFO : Step 37800/50000 rank0 | Train Loss: 8.6845 | Train Loss Unweighted: 2.9779 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6273 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1591 | Regression_eta: 0.0163 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6962 | Classification_binary: 2.6273 | Regression_cos_phi: 0.5315 | Regression_energy: 1.7394 | Regression_eta: 0.7274 | Regression_pt: 1.6788 | Regression_sin_phi: 0.6839 | LR: 2.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4881s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1035 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.3687 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5468.9 | uss_MB: 4997.9 | child_rss_MB: 3195.5 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:27,206] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:38,042] INFO : Step 37900/50000 rank0 | Train Loss: 8.1660 | Train Loss Unweighted: 2.9079 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5796 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1461 | Regression_eta: 0.0164 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6752 | Classification_binary: 2.5796 | Regression_cos_phi: 0.4300 | Regression_energy: 1.5967 | Regression_eta: 0.7319 | Regression_pt: 1.6110 | Regression_sin_phi: 0.5415 | LR: 2.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3712s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0265 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5469.5 | uss_MB: 4997.8 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:57:38,255] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:49,124] INFO : Step 38000/50000 rank0 | Train Loss: 7.7200 | Train Loss Unweighted: 2.9871 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6836 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1333 | Regression_eta: 0.0073 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6539 | Classification_binary: 2.6836 | Regression_cos_phi: 0.4156 | Regression_energy: 1.4575 | Regression_eta: 0.3257 | Regression_pt: 1.5769 | Regression_sin_phi: 0.6067 | LR: 2.22e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4599s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1012 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3662 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5469.5 | uss_MB: 4997.8 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:57:49,337] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:49,366] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 12:57:49,450] INFO : Running validation on rank0 +rank0 [2026-10-28 12:57:49,450] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:57:49,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:57:50,559] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 12:57:50,706] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:57:50,838] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:57:50,913] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:57:50,982] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:57:51,050] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:57:51,120] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:57:51,190] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:57:51,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:57:51,434] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:57:51,566] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:57:51,637] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:57:51,707] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:57:51,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:57:51,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:57:51,916] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:57:52,032] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:57:52,039] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:57:52,041] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:57:52,044] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:57:52,045] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:57:52,048] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:57:52,049] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:57:52,052] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:57:52,055] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:57:52,056] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:57:52,058] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:57:52,061] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:57:52,062] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:57:52,065] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:57:52,067] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:57:52,069] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:57:58,079] INFO : VALIDATION | Step=38000/50000 | Train Loss=7.7200 | Valid Loss=8.1680 | Train Loss Unweighted=2.9871 | Valid Loss Unweighted=2.9460 | Stale=1 | ETA=33.3m +rank0 [2026-10-28 12:57:58,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:57:58,146] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:57:58,151] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:57:58,323] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:57:59,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:58:00,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:58:01,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:58:01,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:58:01,880] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:58:01,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:58:01,938] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:58:01,941] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:58:02,052] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:58:03,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:58:04,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:58:04,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:58:05,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:58:05,399] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:58:05,402] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:58:05,455] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:58:05,458] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:58:05,589] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:58:07,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:58:07,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:58:08,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:58:08,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:58:08,931] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:58:08,938] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:58:09,205] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:58:09,612] INFO : Plotted number of elements +rank0 [2026-10-28 12:58:11,058] INFO : Plotted jets +rank0 [2026-10-28 12:58:12,150] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:12,183] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:58:12,414] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:58:12,807] INFO : Plotted number of elements +rank0 [2026-10-28 12:58:14,729] INFO : Plotted jets +rank0 [2026-10-28 12:58:15,824] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:15,858] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 12:58:16,069] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 12:58:16,432] INFO : Plotted number of elements +rank0 [2026-10-28 12:58:17,808] INFO : Plotted jets +rank0 [2026-10-28 12:58:18,864] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:30,336] INFO : Step 38100/50000 rank0 | Train Loss: 8.2853 | Train Loss Unweighted: 2.9831 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.6430 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1514 | Regression_eta: 0.0121 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6998 | Classification_binary: 2.6430 | Regression_cos_phi: 0.4179 | Regression_energy: 1.6555 | Regression_eta: 0.5417 | Regression_pt: 1.7245 | Regression_sin_phi: 0.6030 | LR: 2.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4504s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0850 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1340 | max_tensorboard_logging: 0.3593 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5654.0 | uss_MB: 5182.9 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:30,552] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:41,437] INFO : Step 38200/50000 rank0 | Train Loss: 8.4095 | Train Loss Unweighted: 2.9581 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.6136 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1542 | Regression_eta: 0.0107 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6893 | Classification_binary: 2.6136 | Regression_cos_phi: 0.5151 | Regression_energy: 1.6853 | Regression_eta: 0.4763 | Regression_pt: 1.7364 | Regression_sin_phi: 0.6936 | LR: 2.18e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4329s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0606 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5654.0 | uss_MB: 5183.0 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:58:41,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:52,436] INFO : Step 38300/50000 rank0 | Train Loss: 8.1590 | Train Loss Unweighted: 2.9319 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.6187 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1357 | Regression_eta: 0.0108 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6670 | Classification_binary: 2.6187 | Regression_cos_phi: 0.5211 | Regression_energy: 1.4835 | Regression_eta: 0.4803 | Regression_pt: 1.5734 | Regression_sin_phi: 0.8151 | LR: 2.16e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4411s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0141 | max_loss: 0.0116 | max_backward: 0.0335 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5543.8 | uss_MB: 5072.9 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:52,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:03,615] INFO : Step 38400/50000 rank0 | Train Loss: 7.6877 | Train Loss Unweighted: 2.8553 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5441 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1371 | Regression_eta: 0.0100 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6736 | Classification_binary: 2.5441 | Regression_cos_phi: 0.3376 | Regression_energy: 1.4991 | Regression_eta: 0.4462 | Regression_pt: 1.6014 | Regression_sin_phi: 0.5857 | LR: 2.14e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4341s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0945 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.3617 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5543.8 | uss_MB: 5073.5 | child_rss_MB: 3200.2 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:03,825] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:14,739] INFO : Step 38500/50000 rank0 | Train Loss: 8.3172 | Train Loss Unweighted: 2.9679 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6263 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1511 | Regression_eta: 0.0102 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7284 | Classification_binary: 2.6263 | Regression_cos_phi: 0.4139 | Regression_energy: 1.6514 | Regression_eta: 0.4549 | Regression_pt: 1.7503 | Regression_sin_phi: 0.6918 | LR: 2.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3742s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.3625 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5544.8 | uss_MB: 5073.6 | child_rss_MB: 3196.2 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:14,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:25,786] INFO : Step 38600/50000 rank0 | Train Loss: 7.7003 | Train Loss Unweighted: 2.8752 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5711 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1335 | Regression_eta: 0.0102 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6900 | Classification_binary: 2.5711 | Regression_cos_phi: 0.4256 | Regression_energy: 1.4600 | Regression_eta: 0.4559 | Regression_pt: 1.5392 | Regression_sin_phi: 0.5585 | LR: 2.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4849s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0473 | max_backward: 0.0332 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3890 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5544.8 | uss_MB: 5073.6 | child_rss_MB: 3197.7 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:59:25,997] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:36,955] INFO : Step 38700/50000 rank0 | Train Loss: 8.0046 | Train Loss Unweighted: 2.9557 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6340 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1404 | Regression_eta: 0.0123 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6732 | Classification_binary: 2.6340 | Regression_cos_phi: 0.3256 | Regression_energy: 1.5349 | Regression_eta: 0.5496 | Regression_pt: 1.6565 | Regression_sin_phi: 0.6307 | LR: 2.09e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0793 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.3833 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5544.8 | uss_MB: 5073.7 | child_rss_MB: 3196.7 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:37,166] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:47,938] INFO : Step 38800/50000 rank0 | Train Loss: 7.9986 | Train Loss Unweighted: 2.8914 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5696 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1377 | Regression_eta: 0.0129 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7049 | Classification_binary: 2.5696 | Regression_cos_phi: 0.3796 | Regression_energy: 1.5058 | Regression_eta: 0.5772 | Regression_pt: 1.6664 | Regression_sin_phi: 0.5950 | LR: 2.07e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0394 | max_backward: 0.0333 | max_optimizer_step: 0.1301 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5544.8 | uss_MB: 5073.8 | child_rss_MB: 3197.8 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:59:48,152] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:59,184] INFO : Step 38900/50000 rank0 | Train Loss: 7.8843 | Train Loss Unweighted: 2.9190 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5996 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1413 | Regression_eta: 0.0088 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7085 | Classification_binary: 2.5996 | Regression_cos_phi: 0.3719 | Regression_energy: 1.5450 | Regression_eta: 0.3905 | Regression_pt: 1.6413 | Regression_sin_phi: 0.6276 | LR: 2.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3910s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0838 | max_forward: 0.0153 | max_loss: 0.0035 | max_backward: 0.0344 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5544.8 | uss_MB: 5072.6 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2852.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:59:59,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:10,298] INFO : Step 39000/50000 rank0 | Train Loss: 7.4532 | Train Loss Unweighted: 2.7462 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4544 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1258 | Regression_eta: 0.0098 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7320 | Classification_binary: 2.4544 | Regression_cos_phi: 0.3759 | Regression_energy: 1.3749 | Regression_eta: 0.4364 | Regression_pt: 1.4815 | Regression_sin_phi: 0.5980 | LR: 2.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3844s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0561 | max_forward: 0.0176 | max_loss: 0.0034 | max_backward: 0.0390 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.2962 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5545.8 | uss_MB: 5075.8 | child_rss_MB: 3195.9 | allocated_MB: 210.7 | reserved_MB: 2852.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:00:10,514] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:10,533] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 13:00:10,616] INFO : Running validation on rank0 +rank0 [2026-10-28 13:00:10,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:00:10,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:00:11,708] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 13:00:11,863] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:00:11,996] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:00:12,070] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:00:12,140] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:00:12,208] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:00:12,277] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:00:12,347] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:00:12,457] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:00:12,590] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:00:12,721] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:00:12,793] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:00:12,863] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:00:12,931] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:00:13,000] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:00:13,070] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:00:13,179] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:00:13,186] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:00:13,188] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:00:13,296] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:00:13,299] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:00:13,301] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:00:13,302] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:00:13,305] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:00:13,308] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:00:13,309] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:00:13,312] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:00:13,314] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:00:13,315] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:00:13,318] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:00:13,320] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:00:13,321] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:00:19,286] INFO : VALIDATION | Step=39000/50000 | Train Loss=7.4532 | Valid Loss=8.0735 | Train Loss Unweighted=2.7462 | Valid Loss Unweighted=2.9264 | Stale=0 | ETA=30.4m +rank0 [2026-10-28 13:00:19,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:19,353] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:00:19,356] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:00:19,529] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:21,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:21,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:22,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:22,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:23,050] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:23,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:23,109] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:00:23,112] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:23,233] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:24,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:25,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:25,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:26,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:26,592] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:26,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:26,650] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:00:26,653] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:00:26,769] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:28,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:28,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:29,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:29,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:30,143] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:30,150] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:00:30,413] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:00:30,822] INFO : Plotted number of elements +rank0 [2026-10-28 13:00:32,271] INFO : Plotted jets +rank0 [2026-10-28 13:00:33,357] INFO : Plotted jet ratio +rank0 [2026-10-28 13:00:33,388] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:33,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:34,016] INFO : Plotted number of elements +rank0 [2026-10-28 13:00:35,413] INFO : Plotted jets +rank0 [2026-10-28 13:00:37,044] INFO : Plotted jet ratio +rank0 [2026-10-28 13:00:37,076] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 13:00:37,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 13:00:37,651] INFO : Plotted number of elements +rank0 [2026-10-28 13:00:39,030] INFO : Plotted jets +rank0 [2026-10-28 13:00:40,090] INFO : Plotted jet ratio +rank0 [2026-10-28 13:00:51,406] INFO : Step 39100/50000 rank0 | Train Loss: 8.3178 | Train Loss Unweighted: 2.9700 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6354 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1473 | Regression_eta: 0.0116 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7533 | Classification_binary: 2.6354 | Regression_cos_phi: 0.4263 | Regression_energy: 1.6102 | Regression_eta: 0.5162 | Regression_pt: 1.6863 | Regression_sin_phi: 0.6902 | LR: 2.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5076s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0167 | max_loss: 0.1081 | max_backward: 0.0383 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3865 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5735.9 | uss_MB: 5265.9 | child_rss_MB: 3193.8 | allocated_MB: 210.7 | reserved_MB: 2856.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:00:51,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:02,623] INFO : Step 39200/50000 rank0 | Train Loss: 8.0174 | Train Loss Unweighted: 2.9415 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6139 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1448 | Regression_eta: 0.0087 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7309 | Classification_binary: 2.6139 | Regression_cos_phi: 0.4513 | Regression_energy: 1.5825 | Regression_eta: 0.3889 | Regression_pt: 1.6849 | Regression_sin_phi: 0.5650 | LR: 2.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4167s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0557 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.3686 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5735.9 | uss_MB: 5263.8 | child_rss_MB: 3196.5 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:02,838] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:13,644] INFO : Step 39300/50000 rank0 | Train Loss: 8.4233 | Train Loss Unweighted: 2.9568 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6039 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1587 | Regression_eta: 0.0096 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6746 | Classification_binary: 2.6039 | Regression_cos_phi: 0.4954 | Regression_energy: 1.7354 | Regression_eta: 0.4265 | Regression_pt: 1.8038 | Regression_sin_phi: 0.6838 | LR: 1.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4850s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0766 | max_backward: 0.0331 | max_optimizer_step: 0.1296 | max_tensorboard_logging: 0.4056 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5616.3 | uss_MB: 5144.5 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:13,856] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:24,845] INFO : Step 39400/50000 rank0 | Train Loss: 7.4922 | Train Loss Unweighted: 2.8894 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5917 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1321 | Regression_eta: 0.0098 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6487 | Classification_binary: 2.5917 | Regression_cos_phi: 0.3331 | Regression_energy: 1.4444 | Regression_eta: 0.4386 | Regression_pt: 1.5201 | Regression_sin_phi: 0.5155 | LR: 1.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3249s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0554 | max_forward: 0.0141 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1308 | max_tensorboard_logging: 0.2774 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5617.3 | uss_MB: 5145.6 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:25,060] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:35,822] INFO : Step 39500/50000 rank0 | Train Loss: 7.9355 | Train Loss Unweighted: 2.8580 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5313 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1480 | Regression_eta: 0.0103 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6459 | Classification_binary: 2.5313 | Regression_cos_phi: 0.4707 | Regression_energy: 1.6177 | Regression_eta: 0.4599 | Regression_pt: 1.6402 | Regression_sin_phi: 0.5698 | LR: 1.94e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.4741s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0980 | max_backward: 0.0333 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5617.3 | uss_MB: 5145.5 | child_rss_MB: 3193.6 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:01:36,036] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:47,087] INFO : Step 39600/50000 rank0 | Train Loss: 7.9733 | Train Loss Unweighted: 2.8895 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5723 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1389 | Regression_eta: 0.0104 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6768 | Classification_binary: 2.5723 | Regression_cos_phi: 0.4617 | Regression_energy: 1.5186 | Regression_eta: 0.4658 | Regression_pt: 1.6129 | Regression_sin_phi: 0.6651 | LR: 1.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3918s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0902 | max_forward: 0.0162 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1324 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5618.3 | uss_MB: 5148.1 | child_rss_MB: 3194.2 | allocated_MB: 210.7 | reserved_MB: 2856.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:01:47,300] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:58,026] INFO : Step 39700/50000 rank0 | Train Loss: 8.5092 | Train Loss Unweighted: 2.8995 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5586 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1500 | Regression_eta: 0.0194 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6866 | Classification_binary: 2.5586 | Regression_cos_phi: 0.4839 | Regression_energy: 1.6401 | Regression_eta: 0.8651 | Regression_pt: 1.6537 | Regression_sin_phi: 0.6212 | LR: 1.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4182s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0289 | max_backward: 0.0333 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.3823 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5618.3 | uss_MB: 5146.1 | child_rss_MB: 3191.7 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:01:58,237] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:09,293] INFO : Step 39800/50000 rank0 | Train Loss: 8.4283 | Train Loss Unweighted: 3.0459 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6965 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1562 | Regression_eta: 0.0118 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7605 | Classification_binary: 2.6965 | Regression_cos_phi: 0.3720 | Regression_energy: 1.7077 | Regression_eta: 0.5258 | Regression_pt: 1.7734 | Regression_sin_phi: 0.5925 | LR: 1.89e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3995s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0925 | max_forward: 0.0185 | max_loss: 0.0034 | max_backward: 0.0394 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.2895 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5619.3 | uss_MB: 5148.3 | child_rss_MB: 3190.8 | allocated_MB: 210.7 | reserved_MB: 2856.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:02:09,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:20,355] INFO : Step 39900/50000 rank0 | Train Loss: 8.0010 | Train Loss Unweighted: 2.7597 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4518 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1326 | Regression_eta: 0.0146 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7201 | Classification_binary: 2.4518 | Regression_cos_phi: 0.4048 | Regression_energy: 1.4501 | Regression_eta: 0.6533 | Regression_pt: 1.5053 | Regression_sin_phi: 0.8156 | LR: 1.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4477s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1320 | max_tensorboard_logging: 0.3788 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5618.0 | uss_MB: 5144.6 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:02:20,571] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:31,433] INFO : Step 40000/50000 rank0 | Train Loss: 8.1728 | Train Loss Unweighted: 2.9375 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5997 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1469 | Regression_eta: 0.0161 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6564 | Classification_binary: 2.5997 | Regression_cos_phi: 0.3214 | Regression_energy: 1.6063 | Regression_eta: 0.7199 | Regression_pt: 1.7440 | Regression_sin_phi: 0.5251 | LR: 1.86e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0813 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3497 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5619.0 | uss_MB: 5145.7 | child_rss_MB: 3192.1 | allocated_MB: 207.0 | reserved_MB: 2856.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:02:31,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:31,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 13:02:31,768] INFO : Running validation on rank0 +rank0 [2026-10-28 13:02:31,768] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:02:31,768] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:02:32,861] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 13:02:33,008] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:02:33,140] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:02:33,215] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:02:33,284] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:02:33,352] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:02:33,421] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:02:33,491] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:02:33,600] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:02:33,733] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:02:33,865] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:02:33,936] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:02:34,006] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:02:34,075] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:02:34,145] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:02:34,214] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:02:34,324] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:02:34,331] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:02:34,334] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:02:34,336] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:02:34,337] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:02:34,340] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:02:34,343] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:02:34,344] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:02:34,347] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:02:34,350] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:02:34,351] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:02:34,354] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:02:34,357] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:02:34,359] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:02:34,360] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:02:34,363] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:02:40,157] INFO : VALIDATION | Step=40000/50000 | Train Loss=8.1728 | Valid Loss=8.0892 | Train Loss Unweighted=2.9375 | Valid Loss Unweighted=2.9256 | Stale=1 | ETA=27.5m +rank0 [2026-10-28 13:02:40,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:02:40,291] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:02:40,295] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:02:40,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:02:42,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:02:42,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:02:43,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:02:44,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:02:44,233] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:02:44,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:02:44,290] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:02:44,294] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:02:44,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:02:45,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:02:46,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:02:47,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:02:47,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:02:47,824] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:02:47,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:02:47,879] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:02:47,882] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:02:48,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:02:49,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:02:50,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:02:50,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:02:51,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:02:51,352] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:02:51,360] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:02:51,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:02:52,027] INFO : Plotted number of elements +rank0 [2026-10-28 13:02:53,476] INFO : Plotted jets +rank0 [2026-10-28 13:02:54,558] INFO : Plotted jet ratio +rank0 [2026-10-28 13:02:54,589] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:02:54,818] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:02:55,210] INFO : Plotted number of elements +rank0 [2026-10-28 13:02:57,141] INFO : Plotted jets +rank0 [2026-10-28 13:02:58,239] INFO : Plotted jet ratio +rank0 [2026-10-28 13:02:58,271] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 13:02:58,486] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 13:02:58,847] INFO : Plotted number of elements +rank0 [2026-10-28 13:03:00,227] INFO : Plotted jets +rank0 [2026-10-28 13:03:01,301] INFO : Plotted jet ratio +rank0 [2026-10-28 13:03:12,453] INFO : Step 40100/50000 rank0 | Train Loss: 7.3796 | Train Loss Unweighted: 2.8598 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.5761 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1244 | Regression_eta: 0.0095 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.5784 | Classification_binary: 2.5761 | Regression_cos_phi: 0.4077 | Regression_energy: 1.3605 | Regression_eta: 0.4243 | Regression_pt: 1.4485 | Regression_sin_phi: 0.5840 | LR: 1.84e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3858s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0139 | max_loss: 0.0034 | max_backward: 0.1218 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5801.9 | uss_MB: 5327.4 | child_rss_MB: 3225.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:03:12,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:23,772] INFO : Step 40200/50000 rank0 | Train Loss: 7.9818 | Train Loss Unweighted: 2.9835 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6717 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1376 | Regression_eta: 0.0085 | Regression_pt: 0.1331 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7140 | Classification_binary: 2.6717 | Regression_cos_phi: 0.3563 | Regression_energy: 1.5040 | Regression_eta: 0.3809 | Regression_pt: 1.5813 | Regression_sin_phi: 0.7737 | LR: 1.83e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4965s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0987 | max_backward: 0.0334 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5801.9 | uss_MB: 5327.6 | child_rss_MB: 3194.6 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:23,984] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:34,674] INFO : Step 40300/50000 rank0 | Train Loss: 7.2253 | Train Loss Unweighted: 2.8859 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.6023 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1255 | Regression_eta: 0.0062 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6369 | Classification_binary: 2.6023 | Regression_cos_phi: 0.3550 | Regression_energy: 1.3719 | Regression_eta: 0.2772 | Regression_pt: 1.4757 | Regression_sin_phi: 0.5064 | LR: 1.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0896 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5801.9 | uss_MB: 5327.4 | child_rss_MB: 3221.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:03:34,890] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:46,055] INFO : Step 40400/50000 rank0 | Train Loss: 7.1136 | Train Loss Unweighted: 2.6160 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3425 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1191 | Regression_eta: 0.0108 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6879 | Classification_binary: 2.3425 | Regression_cos_phi: 0.4172 | Regression_energy: 1.3018 | Regression_eta: 0.4796 | Regression_pt: 1.3452 | Regression_sin_phi: 0.5394 | LR: 1.79e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4704s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1165 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.3516 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5693.0 | uss_MB: 5219.8 | child_rss_MB: 3197.1 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:03:46,267] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:57,096] INFO : Step 40500/50000 rank0 | Train Loss: 8.5539 | Train Loss Unweighted: 2.8966 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5351 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1618 | Regression_eta: 0.0144 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7160 | Classification_binary: 2.5351 | Regression_cos_phi: 0.4158 | Regression_energy: 1.7692 | Regression_eta: 0.6432 | Regression_pt: 1.8169 | Regression_sin_phi: 0.6576 | LR: 1.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5112s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0187 | max_loss: 0.0713 | max_backward: 0.0389 | max_optimizer_step: 0.1324 | max_tensorboard_logging: 0.3871 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5693.0 | uss_MB: 5222.3 | child_rss_MB: 3199.0 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:03:57,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:08,287] INFO : Step 40600/50000 rank0 | Train Loss: 7.9018 | Train Loss Unweighted: 2.7847 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4720 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1383 | Regression_eta: 0.0142 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6668 | Classification_binary: 2.4720 | Regression_cos_phi: 0.3798 | Regression_energy: 1.5121 | Regression_eta: 0.6324 | Regression_pt: 1.5354 | Regression_sin_phi: 0.7033 | LR: 1.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3252s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0209 | max_forward: 0.0159 | max_loss: 0.0035 | max_backward: 0.0382 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5693.0 | uss_MB: 5222.4 | child_rss_MB: 3198.7 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:04:08,503] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:19,321] INFO : Step 40700/50000 rank0 | Train Loss: 7.9515 | Train Loss Unweighted: 2.8729 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5529 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1409 | Regression_eta: 0.0096 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7074 | Classification_binary: 2.5529 | Regression_cos_phi: 0.5130 | Regression_energy: 1.5407 | Regression_eta: 0.4297 | Regression_pt: 1.6226 | Regression_sin_phi: 0.5852 | LR: 1.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4502s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0998 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3435 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5693.0 | uss_MB: 5220.3 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:04:19,534] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:30,358] INFO : Step 40800/50000 rank0 | Train Loss: 7.5936 | Train Loss Unweighted: 2.8754 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5823 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1279 | Regression_eta: 0.0085 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6978 | Classification_binary: 2.5823 | Regression_cos_phi: 0.4779 | Regression_energy: 1.3985 | Regression_eta: 0.3798 | Regression_pt: 1.4816 | Regression_sin_phi: 0.5757 | LR: 1.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0522 | max_backward: 0.0343 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.3728 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5693.0 | uss_MB: 5220.4 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:04:30,574] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:41,459] INFO : Step 40900/50000 rank0 | Train Loss: 8.3018 | Train Loss Unweighted: 2.9003 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5792 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1388 | Regression_eta: 0.0163 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7453 | Classification_binary: 2.5792 | Regression_cos_phi: 0.5053 | Regression_energy: 1.5174 | Regression_eta: 0.7281 | Regression_pt: 1.5610 | Regression_sin_phi: 0.6656 | LR: 1.72e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3964s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0172 | max_loss: 0.0055 | max_backward: 0.0385 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5693.0 | uss_MB: 5222.6 | child_rss_MB: 3198.6 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 13:04:41,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:52,529] INFO : Step 41000/50000 rank0 | Train Loss: 8.6550 | Train Loss Unweighted: 3.0564 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7006 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1555 | Regression_eta: 0.0107 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8521 | Classification_binary: 2.7006 | Regression_cos_phi: 0.4237 | Regression_energy: 1.6996 | Regression_eta: 0.4794 | Regression_pt: 1.8206 | Regression_sin_phi: 0.6791 | LR: 1.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0476 | max_backward: 0.0335 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3701 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5692.3 | uss_MB: 5219.8 | child_rss_MB: 3199.0 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:04:52,741] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:52,758] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 13:04:52,943] INFO : Running validation on rank0 +rank0 [2026-10-28 13:04:52,943] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:04:52,943] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:04:54,044] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 13:04:54,190] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:04:54,324] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:04:54,399] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:04:54,468] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:04:54,537] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:04:54,607] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:04:54,676] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:04:54,785] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:04:54,917] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:04:55,049] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:04:55,121] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:04:55,191] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:04:55,259] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:04:55,329] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:04:55,399] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:04:55,507] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:04:55,514] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:04:55,517] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:04:55,520] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:04:55,522] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:04:55,524] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:04:55,528] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:04:55,530] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:04:55,531] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:04:55,534] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:04:55,535] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:04:55,537] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:04:55,540] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:04:55,543] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:04:55,546] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:04:55,547] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:05:01,607] INFO : VALIDATION | Step=41000/50000 | Train Loss=8.6550 | Valid Loss=8.0760 | Train Loss Unweighted=3.0564 | Valid Loss Unweighted=2.9276 | Stale=2 | ETA=24.7m +rank0 [2026-10-28 13:05:01,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:05:01,708] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:05:01,711] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:05:01,889] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:05:03,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:05:04,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:05:04,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:05:05,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:05:05,411] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:05:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:05:05,468] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:05:05,471] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:05:05,601] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:05:07,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:05:07,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:05:08,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:05:08,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:05:08,983] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:05:08,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:05:09,040] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:05:09,042] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:05:09,163] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:05:10,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:05:11,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:05:11,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:05:12,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:05:12,466] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:05:12,474] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:05:12,734] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:05:13,139] INFO : Plotted number of elements +rank0 [2026-10-28 13:05:14,583] INFO : Plotted jets +rank0 [2026-10-28 13:05:15,668] INFO : Plotted jet ratio +rank0 [2026-10-28 13:05:15,701] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:05:15,931] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:05:16,327] INFO : Plotted number of elements +rank0 [2026-10-28 13:05:17,727] INFO : Plotted jets +rank0 [2026-10-28 13:05:19,386] INFO : Plotted jet ratio +rank0 [2026-10-28 13:05:19,419] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 13:05:19,628] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 13:05:19,989] INFO : Plotted number of elements +rank0 [2026-10-28 13:05:21,366] INFO : Plotted jets +rank0 [2026-10-28 13:05:22,423] INFO : Plotted jet ratio +rank0 [2026-10-28 13:05:33,680] INFO : Step 41100/50000 rank0 | Train Loss: 8.1373 | Train Loss Unweighted: 2.8312 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5183 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1347 | Regression_eta: 0.0121 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7086 | Classification_binary: 2.5183 | Regression_cos_phi: 0.5720 | Regression_energy: 1.4732 | Regression_eta: 0.5395 | Regression_pt: 1.5477 | Regression_sin_phi: 0.7782 | LR: 1.69e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3481s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0503 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0382 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.2741 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5885.6 | uss_MB: 5416.9 | child_rss_MB: 3194.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:05:33,896] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:44,800] INFO : Step 41200/50000 rank0 | Train Loss: 8.3790 | Train Loss Unweighted: 2.8729 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5341 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1496 | Regression_eta: 0.0146 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7315 | Classification_binary: 2.5341 | Regression_cos_phi: 0.5241 | Regression_energy: 1.6352 | Regression_eta: 0.6512 | Regression_pt: 1.6690 | Regression_sin_phi: 0.6340 | LR: 1.67e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0695 | max_forward: 0.0149 | max_loss: 0.0041 | max_backward: 0.0332 | max_optimizer_step: 0.1368 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5885.6 | uss_MB: 5414.8 | child_rss_MB: 3192.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:05:45,017] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:55,777] INFO : Step 41300/50000 rank0 | Train Loss: 8.2107 | Train Loss Unweighted: 2.8333 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5127 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1390 | Regression_eta: 0.0151 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7121 | Classification_binary: 2.5127 | Regression_cos_phi: 0.4974 | Regression_energy: 1.5194 | Regression_eta: 0.6723 | Regression_pt: 1.5726 | Regression_sin_phi: 0.7243 | LR: 1.66e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4399s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0199 | max_backward: 0.0333 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3806 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5885.6 | uss_MB: 5414.8 | child_rss_MB: 3190.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:05:55,996] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:06,958] INFO : Step 41400/50000 rank0 | Train Loss: 7.6895 | Train Loss Unweighted: 2.8100 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5063 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1327 | Regression_eta: 0.0120 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6824 | Classification_binary: 2.5063 | Regression_cos_phi: 0.4628 | Regression_energy: 1.4503 | Regression_eta: 0.5372 | Regression_pt: 1.5231 | Regression_sin_phi: 0.5274 | LR: 1.64e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4366s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0602 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3622 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5763.0 | uss_MB: 5291.4 | child_rss_MB: 3188.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:06:07,172] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:18,054] INFO : Step 41500/50000 rank0 | Train Loss: 7.8747 | Train Loss Unweighted: 2.8599 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5318 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1456 | Regression_eta: 0.0096 | Regression_pt: 0.1429 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6907 | Classification_binary: 2.5318 | Regression_cos_phi: 0.3700 | Regression_energy: 1.5920 | Regression_eta: 0.4278 | Regression_pt: 1.6980 | Regression_sin_phi: 0.5644 | LR: 1.63e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3830s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0193 | max_loss: 0.0036 | max_backward: 0.0336 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5764.0 | uss_MB: 5291.6 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:06:18,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:29,208] INFO : Step 41600/50000 rank0 | Train Loss: 7.8484 | Train Loss Unweighted: 2.8038 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4822 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1430 | Regression_eta: 0.0086 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6642 | Classification_binary: 2.4822 | Regression_cos_phi: 0.4434 | Regression_energy: 1.5632 | Regression_eta: 0.3856 | Regression_pt: 1.6452 | Regression_sin_phi: 0.6646 | LR: 1.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3752s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0503 | max_forward: 0.0144 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5764.0 | uss_MB: 5291.7 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 33.0 +rank0 [2026-10-28 13:06:29,422] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:40,206] INFO : Step 41700/50000 rank0 | Train Loss: 7.9916 | Train Loss Unweighted: 2.9344 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6126 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1439 | Regression_eta: 0.0114 | Regression_pt: 0.1351 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7186 | Classification_binary: 2.6126 | Regression_cos_phi: 0.4079 | Regression_energy: 1.5731 | Regression_eta: 0.5103 | Regression_pt: 1.6059 | Regression_sin_phi: 0.5632 | LR: 1.60e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4323s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0844 | max_loss: 0.0034 | max_backward: 0.0390 | max_optimizer_step: 0.1319 | max_tensorboard_logging: 0.3462 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5764.0 | uss_MB: 5293.8 | child_rss_MB: 3188.2 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:06:40,419] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:51,384] INFO : Step 41800/50000 rank0 | Train Loss: 7.8281 | Train Loss Unweighted: 2.9358 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.6259 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1391 | Regression_eta: 0.0104 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6169 | Classification_binary: 2.6259 | Regression_cos_phi: 0.3886 | Regression_energy: 1.5206 | Regression_eta: 0.4651 | Regression_pt: 1.5570 | Regression_sin_phi: 0.6540 | LR: 1.58e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4128s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0848 | max_forward: 0.0136 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0322 | Memory: rss_MB: 5764.0 | uss_MB: 5291.7 | child_rss_MB: 3190.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:06:51,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:02,321] INFO : Step 41900/50000 rank0 | Train Loss: 7.6639 | Train Loss Unweighted: 2.9224 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.6326 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1270 | Regression_eta: 0.0098 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6250 | Classification_binary: 2.6326 | Regression_cos_phi: 0.4188 | Regression_energy: 1.3882 | Regression_eta: 0.4378 | Regression_pt: 1.4559 | Regression_sin_phi: 0.7054 | LR: 1.57e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5093s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0883 | max_backward: 0.0339 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.3907 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5764.0 | uss_MB: 5291.8 | child_rss_MB: 3190.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:07:02,535] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:13,523] INFO : Step 42000/50000 rank0 | Train Loss: 8.1711 | Train Loss Unweighted: 2.8826 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5551 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1438 | Regression_eta: 0.0126 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7149 | Classification_binary: 2.5551 | Regression_cos_phi: 0.4415 | Regression_energy: 1.5726 | Regression_eta: 0.5632 | Regression_pt: 1.6402 | Regression_sin_phi: 0.6836 | LR: 1.56e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3806s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0939 | max_forward: 0.0137 | max_loss: 0.0036 | max_backward: 0.0337 | max_optimizer_step: 0.1320 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5763.9 | uss_MB: 5291.5 | child_rss_MB: 3188.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:07:13,736] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:13,755] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 13:07:13,837] INFO : Running validation on rank0 +rank0 [2026-10-28 13:07:13,837] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:07:13,837] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:07:14,989] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 13:07:15,137] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:07:15,270] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:07:15,345] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:07:15,413] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:07:15,482] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:07:15,553] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:07:15,624] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:07:15,732] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:07:15,866] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:07:15,999] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:07:16,071] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:07:16,141] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:07:16,210] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:07:16,280] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:07:16,350] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:07:16,462] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:07:16,568] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:07:16,571] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:07:16,575] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:07:16,577] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:07:16,578] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:07:16,581] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:07:16,584] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:07:16,585] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:07:16,588] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:07:16,591] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:07:16,593] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:07:16,594] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:07:16,597] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:07:16,599] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:07:16,600] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:07:22,568] INFO : VALIDATION | Step=42000/50000 | Train Loss=8.1711 | Valid Loss=7.9845 | Train Loss Unweighted=2.8826 | Valid Loss Unweighted=2.8902 | Stale=0 | ETA=21.9m +rank0 [2026-10-28 13:07:22,577] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:07:22,634] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:07:22,637] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:07:22,818] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:07:24,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:07:24,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:07:25,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:07:26,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:07:26,362] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:07:26,365] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:07:26,421] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:07:26,424] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:07:26,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:07:28,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:07:28,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:07:29,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:07:29,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:07:29,933] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:07:29,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:07:29,989] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:07:29,992] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:07:30,109] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:07:31,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:07:32,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:07:32,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:07:33,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:07:33,520] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:07:33,528] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:07:33,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:07:34,195] INFO : Plotted number of elements +rank0 [2026-10-28 13:07:35,638] INFO : Plotted jets +rank0 [2026-10-28 13:07:36,729] INFO : Plotted jet ratio +rank0 [2026-10-28 13:07:36,761] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:07:36,996] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:07:37,390] INFO : Plotted number of elements +rank0 [2026-10-28 13:07:38,784] INFO : Plotted jets +rank0 [2026-10-28 13:07:40,420] INFO : Plotted jet ratio +rank0 [2026-10-28 13:07:40,452] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 13:07:40,659] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 13:07:41,022] INFO : Plotted number of elements +rank0 [2026-10-28 13:07:42,402] INFO : Plotted jets +rank0 [2026-10-28 13:07:43,465] INFO : Plotted jet ratio +rank0 [2026-10-28 13:07:54,893] INFO : Step 42100/50000 rank0 | Train Loss: 7.6065 | Train Loss Unweighted: 2.8897 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5948 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1303 | Regression_eta: 0.0100 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6966 | Classification_binary: 2.5948 | Regression_cos_phi: 0.4638 | Regression_energy: 1.4242 | Regression_eta: 0.4478 | Regression_pt: 1.4672 | Regression_sin_phi: 0.5121 | LR: 1.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4430s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0519 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.3420 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5957.4 | uss_MB: 5486.3 | child_rss_MB: 3192.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:07:55,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:05,858] INFO : Step 42200/50000 rank0 | Train Loss: 8.1083 | Train Loss Unweighted: 2.8238 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5035 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1383 | Regression_eta: 0.0150 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6897 | Classification_binary: 2.5035 | Regression_cos_phi: 0.3945 | Regression_energy: 1.5125 | Regression_eta: 0.6692 | Regression_pt: 1.6006 | Regression_sin_phi: 0.7383 | LR: 1.53e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.4263s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0429 | max_backward: 0.0334 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3697 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5957.4 | uss_MB: 5486.2 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:08:06,074] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:08,526] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 13:08:08,526] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:08:17,436] INFO : Step 42300/50000 rank0 | Train Loss: 7.4707 | Train Loss Unweighted: 2.8143 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5195 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1305 | Regression_eta: 0.0082 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6692 | Classification_binary: 2.5195 | Regression_cos_phi: 0.3910 | Regression_energy: 1.4263 | Regression_eta: 0.3647 | Regression_pt: 1.4964 | Regression_sin_phi: 0.6036 | LR: 1.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4312s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.1401 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5987.6 | uss_MB: 5516.5 | child_rss_MB: 3223.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:08:17,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:28,761] INFO : Step 42400/50000 rank0 | Train Loss: 8.2913 | Train Loss Unweighted: 2.9162 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5942 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1398 | Regression_eta: 0.0182 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7093 | Classification_binary: 2.5942 | Regression_cos_phi: 0.4048 | Regression_energy: 1.5284 | Regression_eta: 0.8122 | Regression_pt: 1.5659 | Regression_sin_phi: 0.6765 | LR: 1.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5081s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0175 | max_loss: 0.0981 | max_backward: 0.0389 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5831.4 | uss_MB: 5362.5 | child_rss_MB: 3197.4 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:08:28,971] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:39,847] INFO : Step 42500/50000 rank0 | Train Loss: 7.7551 | Train Loss Unweighted: 2.8437 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5381 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1339 | Regression_eta: 0.0095 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6842 | Classification_binary: 2.5381 | Regression_cos_phi: 0.4348 | Regression_energy: 1.4644 | Regression_eta: 0.4230 | Regression_pt: 1.5484 | Regression_sin_phi: 0.6622 | LR: 1.49e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4482s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0145 | max_loss: 0.0519 | max_backward: 0.0333 | max_optimizer_step: 0.1337 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5831.4 | uss_MB: 5360.9 | child_rss_MB: 3195.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:08:40,060] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:50,842] INFO : Step 42600/50000 rank0 | Train Loss: 8.1239 | Train Loss Unweighted: 2.9165 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6068 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1339 | Regression_eta: 0.0151 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6559 | Classification_binary: 2.6068 | Regression_cos_phi: 0.4255 | Regression_energy: 1.4642 | Regression_eta: 0.6722 | Regression_pt: 1.5282 | Regression_sin_phi: 0.7712 | LR: 1.48e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4265s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0166 | max_loss: 0.0033 | max_backward: 0.0514 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.2862 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5831.4 | uss_MB: 5361.8 | child_rss_MB: 3193.2 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 13:08:51,055] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:02,028] INFO : Step 42700/50000 rank0 | Train Loss: 7.7339 | Train Loss Unweighted: 2.8490 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5431 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1343 | Regression_eta: 0.0104 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6388 | Classification_binary: 2.5431 | Regression_cos_phi: 0.4663 | Regression_energy: 1.4682 | Regression_eta: 0.4652 | Regression_pt: 1.5533 | Regression_sin_phi: 0.5989 | LR: 1.47e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4216s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0097 | max_backward: 0.0332 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.3770 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5831.4 | uss_MB: 5361.2 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:02,238] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:13,228] INFO : Step 42800/50000 rank0 | Train Loss: 7.6142 | Train Loss Unweighted: 2.8362 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.5373 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1337 | Regression_eta: 0.0128 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6119 | Classification_binary: 2.5373 | Regression_cos_phi: 0.3838 | Regression_energy: 1.4613 | Regression_eta: 0.5700 | Regression_pt: 1.4759 | Regression_sin_phi: 0.5741 | LR: 1.45e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4314s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0755 | max_loss: 0.0037 | max_backward: 0.0335 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3591 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5831.4 | uss_MB: 5361.3 | child_rss_MB: 3196.4 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:09:13,439] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:24,000] INFO : Step 42900/50000 rank0 | Train Loss: 7.5191 | Train Loss Unweighted: 2.8722 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5714 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1325 | Regression_eta: 0.0083 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6391 | Classification_binary: 2.5714 | Regression_cos_phi: 0.3682 | Regression_energy: 1.4482 | Regression_eta: 0.3705 | Regression_pt: 1.5609 | Regression_sin_phi: 0.5607 | LR: 1.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3230s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0706 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.2722 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5831.4 | uss_MB: 5361.3 | child_rss_MB: 3225.3 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:24,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:35,101] INFO : Step 43000/50000 rank0 | Train Loss: 8.0818 | Train Loss Unweighted: 2.9204 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6062 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1372 | Regression_eta: 0.0107 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6932 | Classification_binary: 2.6062 | Regression_cos_phi: 0.4658 | Regression_energy: 1.4998 | Regression_eta: 0.4794 | Regression_pt: 1.5760 | Regression_sin_phi: 0.7613 | LR: 1.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3479s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0035 | max_backward: 0.0752 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.2797 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5830.4 | uss_MB: 5359.1 | child_rss_MB: 3222.8 | allocated_MB: 207.0 | reserved_MB: 2460.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:35,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:35,332] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 13:09:35,415] INFO : Running validation on rank0 +rank0 [2026-10-28 13:09:35,415] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:09:35,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:09:36,596] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 13:09:36,742] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:09:36,875] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:09:36,950] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:09:37,020] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:09:37,088] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:09:37,159] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:09:37,230] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:09:37,342] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:09:37,475] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:09:37,607] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:09:37,680] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:09:37,750] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:09:37,819] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:09:37,890] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:09:37,961] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:09:38,076] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:09:38,083] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:09:38,086] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:09:38,089] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:09:38,091] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:09:38,094] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:09:38,097] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:09:38,099] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:09:38,102] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:09:38,103] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:09:38,105] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:09:38,106] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:09:38,109] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:09:38,112] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:09:38,114] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:09:38,115] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:09:43,921] INFO : VALIDATION | Step=43000/50000 | Train Loss=8.0818 | Valid Loss=7.9589 | Train Loss Unweighted=2.9204 | Valid Loss Unweighted=2.8652 | Stale=0 | ETA=19.1m +rank0 [2026-10-28 13:09:43,930] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:09:43,986] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:09:43,990] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:09:44,169] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:09:45,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:09:46,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:09:46,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:09:47,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:09:47,649] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:09:47,653] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:09:47,707] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:09:47,710] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:09:47,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:09:49,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:09:50,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:09:50,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:09:51,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:09:51,355] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:09:51,358] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:09:51,411] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:09:51,414] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:09:51,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:09:53,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:09:53,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:09:54,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:09:54,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:09:54,997] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:09:55,004] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:09:55,264] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:09:55,675] INFO : Plotted number of elements +rank0 [2026-10-28 13:09:57,120] INFO : Plotted jets +rank0 [2026-10-28 13:09:58,211] INFO : Plotted jet ratio +rank0 [2026-10-28 13:09:58,244] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:09:58,478] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:09:58,872] INFO : Plotted number of elements +rank0 [2026-10-28 13:10:00,841] INFO : Plotted jets +rank0 [2026-10-28 13:10:01,940] INFO : Plotted jet ratio +rank0 [2026-10-28 13:10:01,972] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 13:10:02,181] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 13:10:02,545] INFO : Plotted number of elements +rank0 [2026-10-28 13:10:03,923] INFO : Plotted jets +rank0 [2026-10-28 13:10:04,982] INFO : Plotted jet ratio +rank0 [2026-10-28 13:10:16,224] INFO : Step 43100/50000 rank0 | Train Loss: 7.6511 | Train Loss Unweighted: 2.8176 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5278 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1254 | Regression_eta: 0.0131 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6466 | Classification_binary: 2.5278 | Regression_cos_phi: 0.4357 | Regression_energy: 1.3709 | Regression_eta: 0.5820 | Regression_pt: 1.4320 | Regression_sin_phi: 0.6560 | LR: 1.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3856s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.3324 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6032.3 | uss_MB: 5563.0 | child_rss_MB: 3197.8 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:16,436] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:27,310] INFO : Step 43200/50000 rank0 | Train Loss: 7.9706 | Train Loss Unweighted: 2.8162 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5103 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1329 | Regression_eta: 0.0149 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6748 | Classification_binary: 2.5103 | Regression_cos_phi: 0.4887 | Regression_energy: 1.4524 | Regression_eta: 0.6632 | Regression_pt: 1.4904 | Regression_sin_phi: 0.6908 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4956s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1153 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3542 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6032.3 | uss_MB: 5562.9 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:10:27,523] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:38,330] INFO : Step 43300/50000 rank0 | Train Loss: 7.7773 | Train Loss Unweighted: 2.8255 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5288 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1281 | Regression_eta: 0.0131 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6793 | Classification_binary: 2.5288 | Regression_cos_phi: 0.4785 | Regression_energy: 1.4008 | Regression_eta: 0.5854 | Regression_pt: 1.4656 | Regression_sin_phi: 0.6389 | LR: 1.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4627s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0184 | max_loss: 0.0298 | max_backward: 0.0393 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6032.3 | uss_MB: 5565.0 | child_rss_MB: 3191.6 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:10:38,544] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:49,515] INFO : Step 43400/50000 rank0 | Train Loss: 8.1726 | Train Loss Unweighted: 2.9588 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6436 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1384 | Regression_eta: 0.0120 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7457 | Classification_binary: 2.6436 | Regression_cos_phi: 0.4750 | Regression_energy: 1.5132 | Regression_eta: 0.5372 | Regression_pt: 1.5447 | Regression_sin_phi: 0.7132 | LR: 1.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5157s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1223 | max_loss: 0.0036 | max_backward: 0.0384 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.3751 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5908.3 | uss_MB: 5440.4 | child_rss_MB: 3195.3 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:10:49,727] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:00,592] INFO : Step 43500/50000 rank0 | Train Loss: 8.6365 | Train Loss Unweighted: 2.8841 | Train Loss Components: Classification: 0.0156 | Classification_binary: 2.5406 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1510 | Regression_eta: 0.0250 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.5871 | Classification_binary: 2.5406 | Regression_cos_phi: 0.4640 | Regression_energy: 1.6508 | Regression_eta: 1.1136 | Regression_pt: 1.6396 | Regression_sin_phi: 0.6408 | LR: 1.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0562 | max_loss: 0.0035 | max_backward: 0.0380 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3365 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5909.3 | uss_MB: 5441.6 | child_rss_MB: 3195.5 | allocated_MB: 210.7 | reserved_MB: 2584.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:11:00,806] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:11,713] INFO : Step 43600/50000 rank0 | Train Loss: 8.5140 | Train Loss Unweighted: 2.9939 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.6513 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1520 | Regression_eta: 0.0174 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6734 | Classification_binary: 2.6513 | Regression_cos_phi: 0.4745 | Regression_energy: 1.6623 | Regression_eta: 0.7769 | Regression_pt: 1.6835 | Regression_sin_phi: 0.5922 | LR: 1.36e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3559s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0549 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5909.3 | uss_MB: 5439.9 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:11:11,930] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:22,760] INFO : Step 43700/50000 rank0 | Train Loss: 7.9316 | Train Loss Unweighted: 2.9018 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5808 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1437 | Regression_eta: 0.0099 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6633 | Classification_binary: 2.5808 | Regression_cos_phi: 0.4141 | Regression_energy: 1.5713 | Regression_eta: 0.4421 | Regression_pt: 1.6236 | Regression_sin_phi: 0.6364 | LR: 1.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4903s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1216 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3531 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5910.3 | uss_MB: 5440.4 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:22,976] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:33,866] INFO : Step 43800/50000 rank0 | Train Loss: 8.1004 | Train Loss Unweighted: 2.8529 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5341 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1408 | Regression_eta: 0.0129 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7080 | Classification_binary: 2.5341 | Regression_cos_phi: 0.4723 | Regression_energy: 1.5390 | Regression_eta: 0.5764 | Regression_pt: 1.5629 | Regression_sin_phi: 0.7078 | LR: 1.34e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3793s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.0344 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3419 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5910.3 | uss_MB: 5440.6 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:34,078] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:45,000] INFO : Step 43900/50000 rank0 | Train Loss: 8.2224 | Train Loss Unweighted: 2.8321 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5034 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1450 | Regression_eta: 0.0153 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6616 | Classification_binary: 2.5034 | Regression_cos_phi: 0.5384 | Regression_energy: 1.5853 | Regression_eta: 0.6811 | Regression_pt: 1.6146 | Regression_sin_phi: 0.6380 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3311s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0227 | max_forward: 0.0144 | max_loss: 0.0036 | max_backward: 0.0338 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5911.3 | uss_MB: 5440.8 | child_rss_MB: 3190.2 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:45,216] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:56,060] INFO : Step 44000/50000 rank0 | Train Loss: 8.3459 | Train Loss Unweighted: 2.8644 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.5309 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1425 | Regression_eta: 0.0230 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6379 | Classification_binary: 2.5309 | Regression_cos_phi: 0.3872 | Regression_energy: 1.5574 | Regression_eta: 1.0275 | Regression_pt: 1.6548 | Regression_sin_phi: 0.5502 | LR: 1.32e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3988s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0223 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5909.5 | uss_MB: 5438.4 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2584.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:11:56,276] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:56,295] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 13:11:56,379] INFO : Running validation on rank0 +rank0 [2026-10-28 13:11:56,379] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:11:56,379] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:11:57,484] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 13:11:57,631] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:11:57,764] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:11:57,838] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:11:57,907] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:11:57,975] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:11:58,045] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:11:58,115] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:11:58,226] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:11:58,359] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:11:58,491] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:11:58,563] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:11:58,633] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:11:58,702] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:11:58,772] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:11:58,842] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:11:58,961] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:11:58,968] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:11:58,970] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:11:58,971] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:11:58,974] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:11:58,977] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:11:58,980] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:11:58,982] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:11:58,985] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:11:58,986] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:11:59,096] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:11:59,099] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:11:59,102] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:11:59,104] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:11:59,107] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:11:59,108] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:12:04,949] INFO : VALIDATION | Step=44000/50000 | Train Loss=8.3459 | Valid Loss=7.9725 | Train Loss Unweighted=2.8644 | Valid Loss Unweighted=2.8735 | Stale=1 | ETA=16.3m +rank0 [2026-10-28 13:12:04,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:12:05,014] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:12:05,018] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:12:05,197] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:12:06,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:12:07,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:12:07,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:12:08,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:12:08,663] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:12:08,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:12:08,721] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:12:08,724] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:12:08,846] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:12:10,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:12:10,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:12:11,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:12:12,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:12:12,219] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:12:12,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:12:12,276] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:12:12,278] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:12:12,402] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:12:14,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:12:14,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:12:15,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:12:15,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:12:15,817] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:12:15,825] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:12:16,092] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:12:16,502] INFO : Plotted number of elements +rank0 [2026-10-28 13:12:17,946] INFO : Plotted jets +rank0 [2026-10-28 13:12:19,034] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:19,067] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:12:19,298] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:12:19,695] INFO : Plotted number of elements +rank0 [2026-10-28 13:12:21,087] INFO : Plotted jets +rank0 [2026-10-28 13:12:22,756] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:22,787] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 13:12:22,997] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 13:12:23,361] INFO : Plotted number of elements +rank0 [2026-10-28 13:12:24,744] INFO : Plotted jets +rank0 [2026-10-28 13:12:25,808] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:37,210] INFO : Step 44100/50000 rank0 | Train Loss: 8.4101 | Train Loss Unweighted: 2.9574 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.6243 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1468 | Regression_eta: 0.0179 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6804 | Classification_binary: 2.6243 | Regression_cos_phi: 0.4518 | Regression_energy: 1.6051 | Regression_eta: 0.7961 | Regression_pt: 1.6259 | Regression_sin_phi: 0.6265 | LR: 1.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4925s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1192 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3381 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6097.0 | uss_MB: 5626.9 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:37,421] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:48,281] INFO : Step 44200/50000 rank0 | Train Loss: 8.8956 | Train Loss Unweighted: 2.9259 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5751 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1541 | Regression_eta: 0.0296 | Regression_pt: 0.1355 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7158 | Classification_binary: 2.5751 | Regression_cos_phi: 0.4299 | Regression_energy: 1.6851 | Regression_eta: 1.3217 | Regression_pt: 1.6098 | Regression_sin_phi: 0.5582 | LR: 1.30e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4542s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1063 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.3284 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 6097.0 | uss_MB: 5627.0 | child_rss_MB: 3196.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:48,494] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:59,437] INFO : Step 44300/50000 rank0 | Train Loss: 7.7092 | Train Loss Unweighted: 2.7963 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5001 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1306 | Regression_eta: 0.0123 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7057 | Classification_binary: 2.5001 | Regression_cos_phi: 0.4432 | Regression_energy: 1.4277 | Regression_eta: 0.5483 | Regression_pt: 1.4359 | Regression_sin_phi: 0.6483 | LR: 1.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3530s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0416 | max_forward: 0.0158 | max_loss: 0.0037 | max_backward: 0.0342 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6097.0 | uss_MB: 5627.0 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:59,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:10,427] INFO : Step 44400/50000 rank0 | Train Loss: 7.9654 | Train Loss Unweighted: 2.8240 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5000 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1414 | Regression_eta: 0.0126 | Regression_pt: 0.1388 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6828 | Classification_binary: 2.5000 | Regression_cos_phi: 0.4595 | Regression_energy: 1.5463 | Regression_eta: 0.5630 | Regression_pt: 1.6488 | Regression_sin_phi: 0.5651 | LR: 1.28e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4481s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0036 | max_backward: 0.0338 | max_optimizer_step: 0.1369 | max_tensorboard_logging: 0.3974 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5983.6 | uss_MB: 5513.1 | child_rss_MB: 3193.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:13:10,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:21,623] INFO : Step 44500/50000 rank0 | Train Loss: 8.1268 | Train Loss Unweighted: 2.8547 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5371 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1394 | Regression_eta: 0.0140 | Regression_pt: 0.1302 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7419 | Classification_binary: 2.5371 | Regression_cos_phi: 0.4233 | Regression_energy: 1.5245 | Regression_eta: 0.6256 | Regression_pt: 1.5475 | Regression_sin_phi: 0.7270 | LR: 1.27e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4596s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0845 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3371 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5984.6 | uss_MB: 5514.2 | child_rss_MB: 3190.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:13:21,837] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:32,666] INFO : Step 44600/50000 rank0 | Train Loss: 8.3798 | Train Loss Unweighted: 2.7587 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4114 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1488 | Regression_eta: 0.0188 | Regression_pt: 0.1478 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7230 | Classification_binary: 2.4114 | Regression_cos_phi: 0.3937 | Regression_energy: 1.6270 | Regression_eta: 0.8383 | Regression_pt: 1.7558 | Regression_sin_phi: 0.6306 | LR: 1.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0172 | max_backward: 0.0338 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5984.6 | uss_MB: 5514.7 | child_rss_MB: 3192.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:13:32,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:43,865] INFO : Step 44700/50000 rank0 | Train Loss: 8.0817 | Train Loss Unweighted: 2.8372 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5158 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1404 | Regression_eta: 0.0150 | Regression_pt: 0.1343 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6768 | Classification_binary: 2.5158 | Regression_cos_phi: 0.4557 | Regression_energy: 1.5349 | Regression_eta: 0.6682 | Regression_pt: 1.5962 | Regression_sin_phi: 0.6340 | LR: 1.25e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3883s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0292 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.3337 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5984.6 | uss_MB: 5514.8 | child_rss_MB: 3195.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:13:44,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:55,010] INFO : Step 44800/50000 rank0 | Train Loss: 7.7122 | Train Loss Unweighted: 2.8065 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4914 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1401 | Regression_eta: 0.0106 | Regression_pt: 0.1343 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6981 | Classification_binary: 2.4914 | Regression_cos_phi: 0.3535 | Regression_energy: 1.5321 | Regression_eta: 0.4727 | Regression_pt: 1.5957 | Regression_sin_phi: 0.5687 | LR: 1.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4387s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0019 | max_forward: 0.0719 | max_loss: 0.0035 | max_backward: 0.0387 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.3477 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5984.6 | uss_MB: 5517.3 | child_rss_MB: 3196.0 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:13:55,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:06,027] INFO : Step 44900/50000 rank0 | Train Loss: 8.1321 | Train Loss Unweighted: 2.8843 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5749 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1337 | Regression_eta: 0.0122 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6784 | Classification_binary: 2.5749 | Regression_cos_phi: 0.6306 | Regression_energy: 1.4619 | Regression_eta: 0.5427 | Regression_pt: 1.5222 | Regression_sin_phi: 0.7214 | LR: 1.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4607s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0015 | max_forward: 0.0144 | max_loss: 0.0536 | max_backward: 0.0334 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.3813 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5984.6 | uss_MB: 5515.2 | child_rss_MB: 3196.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:06,238] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:17,172] INFO : Step 45000/50000 rank0 | Train Loss: 7.8200 | Train Loss Unweighted: 2.9315 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.6181 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1375 | Regression_eta: 0.0099 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6491 | Classification_binary: 2.6181 | Regression_cos_phi: 0.4011 | Regression_energy: 1.5034 | Regression_eta: 0.4400 | Regression_pt: 1.6210 | Regression_sin_phi: 0.5874 | LR: 1.22e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4616s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0267 | max_backward: 0.0335 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.3779 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5984.6 | uss_MB: 5515.2 | child_rss_MB: 3192.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:17,384] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:17,498] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 13:14:17,582] INFO : Running validation on rank0 +rank0 [2026-10-28 13:14:17,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:14:17,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:14:18,686] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 13:14:18,834] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:14:18,967] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:14:19,040] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:14:19,109] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:14:19,178] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:14:19,248] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:14:19,317] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:14:19,428] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:14:19,562] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:14:19,695] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:14:19,767] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:14:19,836] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:14:19,906] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:14:19,976] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:14:20,047] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:14:20,155] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:14:20,162] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:14:20,165] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:14:20,168] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:14:20,171] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:14:20,172] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:14:20,174] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:14:20,177] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:14:20,178] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:14:20,181] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:14:20,183] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:14:20,186] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:14:20,188] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:14:20,190] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:14:20,296] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:14:20,299] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:14:26,162] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.8200 | Valid Loss=7.9041 | Train Loss Unweighted=2.9315 | Valid Loss Unweighted=2.8544 | Stale=0 | ETA=13.5m +rank0 [2026-10-28 13:14:26,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:26,228] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:14:26,231] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:14:26,412] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:14:27,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:14:28,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:14:29,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:14:29,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:14:29,907] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:14:29,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:29,966] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:14:29,969] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:14:30,085] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:14:31,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:14:32,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:14:32,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:14:33,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:14:33,701] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:14:33,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:33,758] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:14:33,761] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:14:33,890] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:14:35,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:14:35,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:14:36,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:14:37,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:14:37,236] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:14:37,243] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:14:37,506] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:14:37,917] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:39,361] INFO : Plotted jets +rank0 [2026-10-28 13:14:40,459] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:40,491] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:14:40,730] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:14:41,121] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:42,516] INFO : Plotted jets +rank0 [2026-10-28 13:14:44,185] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:44,218] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 13:14:44,428] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 13:14:44,792] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:46,174] INFO : Plotted jets +rank0 [2026-10-28 13:14:47,249] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:58,628] INFO : Step 45100/50000 rank0 | Train Loss: 8.0934 | Train Loss Unweighted: 2.7238 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4080 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1364 | Regression_eta: 0.0156 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7362 | Classification_binary: 2.4080 | Regression_cos_phi: 0.5595 | Regression_energy: 1.4911 | Regression_eta: 0.6970 | Regression_pt: 1.5267 | Regression_sin_phi: 0.6750 | LR: 1.21e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4586s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0169 | max_loss: 0.0035 | max_backward: 0.0383 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6172.3 | uss_MB: 5705.3 | child_rss_MB: 3188.5 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 13:14:58,842] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:09,808] INFO : Step 45200/50000 rank0 | Train Loss: 8.4949 | Train Loss Unweighted: 2.9645 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.6296 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1481 | Regression_eta: 0.0161 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6654 | Classification_binary: 2.6296 | Regression_cos_phi: 0.4551 | Regression_energy: 1.6190 | Regression_eta: 0.7175 | Regression_pt: 1.6377 | Regression_sin_phi: 0.7706 | LR: 1.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3939s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0035 | max_backward: 0.0340 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3329 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6172.3 | uss_MB: 5703.3 | child_rss_MB: 3194.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:15:10,022] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:20,918] INFO : Step 45300/50000 rank0 | Train Loss: 7.9847 | Train Loss Unweighted: 2.9184 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.5979 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1430 | Regression_eta: 0.0077 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7007 | Classification_binary: 2.5979 | Regression_cos_phi: 0.4674 | Regression_energy: 1.5630 | Regression_eta: 0.3431 | Regression_pt: 1.6238 | Regression_sin_phi: 0.6887 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4848s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1019 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1383 | max_tensorboard_logging: 0.3377 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6172.3 | uss_MB: 5703.3 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:21,131] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:32,057] INFO : Step 45400/50000 rank0 | Train Loss: 7.3751 | Train Loss Unweighted: 2.7944 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.5013 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1303 | Regression_eta: 0.0084 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6398 | Classification_binary: 2.5013 | Regression_cos_phi: 0.4017 | Regression_energy: 1.4240 | Regression_eta: 0.3732 | Regression_pt: 1.4920 | Regression_sin_phi: 0.5430 | LR: 1.19e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1126 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3384 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6172.3 | uss_MB: 5703.3 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:32,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:43,054] INFO : Step 45500/50000 rank0 | Train Loss: 7.3840 | Train Loss Unweighted: 2.7373 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4482 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1248 | Regression_eta: 0.0089 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6845 | Classification_binary: 2.4482 | Regression_cos_phi: 0.3835 | Regression_energy: 1.3640 | Regression_eta: 0.3979 | Regression_pt: 1.4821 | Regression_sin_phi: 0.6237 | LR: 1.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4645s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0052 | max_backward: 0.0331 | max_optimizer_step: 0.1368 | max_tensorboard_logging: 0.3982 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6053.2 | uss_MB: 5585.0 | child_rss_MB: 3188.8 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:15:43,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:54,222] INFO : Step 45600/50000 rank0 | Train Loss: 8.1330 | Train Loss Unweighted: 2.7890 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4483 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1510 | Regression_eta: 0.0120 | Regression_pt: 0.1452 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7292 | Classification_binary: 2.4483 | Regression_cos_phi: 0.4262 | Regression_energy: 1.6512 | Regression_eta: 0.5335 | Regression_pt: 1.7258 | Regression_sin_phi: 0.6189 | LR: 1.17e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4844s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1015 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1367 | max_tensorboard_logging: 0.3435 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6053.2 | uss_MB: 5585.2 | child_rss_MB: 3192.2 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:54,438] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:05,311] INFO : Step 45700/50000 rank0 | Train Loss: 8.3422 | Train Loss Unweighted: 2.8696 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5177 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1582 | Regression_eta: 0.0120 | Regression_pt: 0.1498 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6701 | Classification_binary: 2.5177 | Regression_cos_phi: 0.4934 | Regression_energy: 1.7293 | Regression_eta: 0.5365 | Regression_pt: 1.7797 | Regression_sin_phi: 0.6155 | LR: 1.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3842s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3227 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6053.2 | uss_MB: 5585.2 | child_rss_MB: 3187.8 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:05,525] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:16,449] INFO : Step 45800/50000 rank0 | Train Loss: 8.5766 | Train Loss Unweighted: 2.8694 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5458 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1378 | Regression_eta: 0.0165 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7052 | Classification_binary: 2.5458 | Regression_cos_phi: 0.6524 | Regression_energy: 1.5067 | Regression_eta: 0.7351 | Regression_pt: 1.5610 | Regression_sin_phi: 0.8703 | LR: 1.16e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4250s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1034 | max_forward: 0.0139 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6053.2 | uss_MB: 5585.3 | child_rss_MB: 3190.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:16:16,664] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:27,477] INFO : Step 45900/50000 rank0 | Train Loss: 8.3949 | Train Loss Unweighted: 2.9452 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.6034 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1508 | Regression_eta: 0.0115 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7076 | Classification_binary: 2.6034 | Regression_cos_phi: 0.4580 | Regression_energy: 1.6487 | Regression_eta: 0.5137 | Regression_pt: 1.7336 | Regression_sin_phi: 0.7299 | LR: 1.15e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4893s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1212 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3471 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6054.2 | uss_MB: 5585.8 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:27,693] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:38,604] INFO : Step 46000/50000 rank0 | Train Loss: 8.3493 | Train Loss Unweighted: 2.8210 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4917 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1439 | Regression_eta: 0.0185 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6737 | Classification_binary: 2.4917 | Regression_cos_phi: 0.5224 | Regression_energy: 1.5731 | Regression_eta: 0.8266 | Regression_pt: 1.5902 | Regression_sin_phi: 0.6717 | LR: 1.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4054s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0368 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.3353 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6054.2 | uss_MB: 5585.8 | child_rss_MB: 3190.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:38,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:38,846] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 13:16:38,930] INFO : Running validation on rank0 +rank0 [2026-10-28 13:16:38,930] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:16:38,930] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:16:40,030] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 13:16:40,177] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:16:40,313] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:16:40,388] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:16:40,458] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:16:40,526] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:16:40,596] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:16:40,667] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:16:40,778] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:16:40,912] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:16:41,044] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:16:41,116] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:16:41,185] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:16:41,254] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:16:41,324] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:16:41,395] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:16:41,511] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:16:41,518] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:16:41,521] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:16:41,524] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:16:41,525] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:16:41,528] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:16:41,636] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:16:41,639] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:16:41,642] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:16:41,643] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:16:41,645] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:16:41,648] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:16:41,649] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:16:41,652] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:16:41,655] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:16:41,658] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:16:47,584] INFO : VALIDATION | Step=46000/50000 | Train Loss=8.3493 | Valid Loss=7.9248 | Train Loss Unweighted=2.8210 | Valid Loss Unweighted=2.8603 | Stale=1 | ETA=10.8m +rank0 [2026-10-28 13:16:47,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:16:47,648] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:16:47,651] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:16:47,834] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:16:49,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:16:50,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:16:50,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:16:51,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:16:51,512] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:16:51,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:16:51,570] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:16:51,573] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:16:51,699] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:16:53,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:16:53,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:16:54,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:16:55,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:16:55,254] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:16:55,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:16:55,311] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:16:55,314] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:16:55,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:16:56,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:16:57,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:16:58,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:16:58,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:16:58,668] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:16:58,675] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:16:58,936] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:16:59,345] INFO : Plotted number of elements +rank0 [2026-10-28 13:17:00,796] INFO : Plotted jets +rank0 [2026-10-28 13:17:01,896] INFO : Plotted jet ratio +rank0 [2026-10-28 13:17:01,928] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:17:02,158] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:17:02,553] INFO : Plotted number of elements +rank0 [2026-10-28 13:17:03,954] INFO : Plotted jets +rank0 [2026-10-28 13:17:05,637] INFO : Plotted jet ratio +rank0 [2026-10-28 13:17:05,669] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 13:17:05,874] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 13:17:06,238] INFO : Plotted number of elements +rank0 [2026-10-28 13:17:07,624] INFO : Plotted jets +rank0 [2026-10-28 13:17:08,699] INFO : Plotted jet ratio +rank0 [2026-10-28 13:17:20,145] INFO : Step 46100/50000 rank0 | Train Loss: 7.9236 | Train Loss Unweighted: 2.8441 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5189 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1440 | Regression_eta: 0.0127 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6982 | Classification_binary: 2.5189 | Regression_cos_phi: 0.3615 | Regression_energy: 1.5744 | Regression_eta: 0.5675 | Regression_pt: 1.6446 | Regression_sin_phi: 0.5585 | LR: 1.13e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0542 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1369 | max_tensorboard_logging: 0.3316 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6249.5 | uss_MB: 5780.2 | child_rss_MB: 3189.0 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:20,356] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:31,485] INFO : Step 46200/50000 rank0 | Train Loss: 7.5671 | Train Loss Unweighted: 2.8368 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.5311 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1347 | Regression_eta: 0.0092 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6503 | Classification_binary: 2.5311 | Regression_cos_phi: 0.3654 | Regression_energy: 1.4729 | Regression_eta: 0.4105 | Regression_pt: 1.5800 | Regression_sin_phi: 0.5568 | LR: 1.13e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3593s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1536 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6249.5 | uss_MB: 5780.2 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:17:31,697] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:42,420] INFO : Step 46300/50000 rank0 | Train Loss: 7.8275 | Train Loss Unweighted: 2.8179 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.4919 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1452 | Regression_eta: 0.0088 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6264 | Classification_binary: 2.4919 | Regression_cos_phi: 0.4099 | Regression_energy: 1.5877 | Regression_eta: 0.3921 | Regression_pt: 1.6925 | Regression_sin_phi: 0.6270 | LR: 1.12e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3552s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6249.5 | uss_MB: 5780.2 | child_rss_MB: 3192.3 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:42,640] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:53,518] INFO : Step 46400/50000 rank0 | Train Loss: 7.7274 | Train Loss Unweighted: 2.8074 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.5025 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1355 | Regression_eta: 0.0100 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7227 | Classification_binary: 2.5025 | Regression_cos_phi: 0.4556 | Regression_energy: 1.4813 | Regression_eta: 0.4470 | Regression_pt: 1.5060 | Regression_sin_phi: 0.6123 | LR: 1.11e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4427s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1010 | max_forward: 0.0159 | max_loss: 0.0034 | max_backward: 0.0382 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6249.5 | uss_MB: 5782.3 | child_rss_MB: 3193.8 | allocated_MB: 210.7 | reserved_MB: 2814.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:17:53,734] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:04,536] INFO : Step 46500/50000 rank0 | Train Loss: 8.3743 | Train Loss Unweighted: 2.9927 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6670 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1406 | Regression_eta: 0.0145 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7244 | Classification_binary: 2.6670 | Regression_cos_phi: 0.4850 | Regression_energy: 1.5372 | Regression_eta: 0.6486 | Regression_pt: 1.6225 | Regression_sin_phi: 0.6895 | LR: 1.11e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3809s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.3250 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6128.5 | uss_MB: 5659.8 | child_rss_MB: 3194.2 | allocated_MB: 207.0 | reserved_MB: 2814.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:18:04,751] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:15,654] INFO : Step 46600/50000 rank0 | Train Loss: 7.3867 | Train Loss Unweighted: 2.7142 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4266 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1272 | Regression_eta: 0.0092 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6901 | Classification_binary: 2.4266 | Regression_cos_phi: 0.3956 | Regression_energy: 1.3901 | Regression_eta: 0.4101 | Regression_pt: 1.4255 | Regression_sin_phi: 0.6487 | LR: 1.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4850s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1208 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1352 | max_tensorboard_logging: 0.3378 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6129.5 | uss_MB: 5660.3 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:18:15,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:26,797] INFO : Step 46700/50000 rank0 | Train Loss: 8.6452 | Train Loss Unweighted: 3.0321 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6801 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1536 | Regression_eta: 0.0127 | Regression_pt: 0.1512 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7399 | Classification_binary: 2.6801 | Regression_cos_phi: 0.5023 | Regression_energy: 1.6788 | Regression_eta: 0.5645 | Regression_pt: 1.7961 | Regression_sin_phi: 0.6834 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3430s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0271 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6129.5 | uss_MB: 5660.4 | child_rss_MB: 3196.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:18:27,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:37,846] INFO : Step 46800/50000 rank0 | Train Loss: 7.5495 | Train Loss Unweighted: 2.7842 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5016 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1206 | Regression_eta: 0.0136 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6614 | Classification_binary: 2.5016 | Regression_cos_phi: 0.4530 | Regression_energy: 1.3190 | Regression_eta: 0.6067 | Regression_pt: 1.3939 | Regression_sin_phi: 0.6139 | LR: 1.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4431s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0676 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3294 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6129.5 | uss_MB: 5660.7 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:18:38,060] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:48,904] INFO : Step 46900/50000 rank0 | Train Loss: 7.7648 | Train Loss Unweighted: 2.8401 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5413 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1299 | Regression_eta: 0.0121 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7104 | Classification_binary: 2.5413 | Regression_cos_phi: 0.4596 | Regression_energy: 1.4204 | Regression_eta: 0.5419 | Regression_pt: 1.4765 | Regression_sin_phi: 0.6146 | LR: 1.09e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0172 | max_loss: 0.0594 | max_backward: 0.0387 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.3876 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6129.5 | uss_MB: 5663.2 | child_rss_MB: 3197.9 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:18:49,117] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:00,083] INFO : Step 47000/50000 rank0 | Train Loss: 7.5365 | Train Loss Unweighted: 2.8532 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5705 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1209 | Regression_eta: 0.0103 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7614 | Classification_binary: 2.5705 | Regression_cos_phi: 0.4080 | Regression_energy: 1.3218 | Regression_eta: 0.4590 | Regression_pt: 1.4083 | Regression_sin_phi: 0.6074 | LR: 1.08e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4671s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1097 | max_loss: 0.0036 | max_backward: 0.0351 | max_optimizer_step: 0.1370 | max_tensorboard_logging: 0.3305 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6129.5 | uss_MB: 5661.5 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:19:00,296] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:00,333] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 13:19:00,425] INFO : Running validation on rank0 +rank0 [2026-10-28 13:19:00,426] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:19:00,426] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:19:01,528] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 13:19:01,676] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:19:01,809] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:19:01,883] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:19:01,966] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:19:02,040] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:19:02,110] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:19:02,180] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:19:02,307] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:19:02,441] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:19:02,574] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:19:02,646] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:19:02,716] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:19:02,786] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:19:02,857] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:19:02,927] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:19:03,045] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:19:03,052] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:19:03,055] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:19:03,058] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:19:03,059] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:19:03,062] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:19:03,063] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:19:03,066] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:19:03,168] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:19:03,171] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:19:03,174] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:19:03,177] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:19:03,178] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:19:03,180] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:19:03,183] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:19:03,184] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:19:09,037] INFO : VALIDATION | Step=47000/50000 | Train Loss=7.5365 | Valid Loss=7.9068 | Train Loss Unweighted=2.8532 | Valid Loss Unweighted=2.8517 | Stale=2 | ETA=8.1m +rank0 [2026-10-28 13:19:09,046] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:19:09,103] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:19:09,107] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:19:09,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:19:10,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:19:11,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:19:12,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:19:12,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:19:12,757] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:19:12,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:19:12,817] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:19:12,820] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:12,942] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:19:14,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:19:15,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:19:15,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:19:16,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:19:16,460] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:19:16,463] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:19:16,517] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:19:16,520] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:19:16,641] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:19:18,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:19:18,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:19:19,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:19:19,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:19:20,118] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:19:20,126] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:19:20,385] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:19:20,796] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:22,240] INFO : Plotted jets +rank0 [2026-10-28 13:19:23,324] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:23,356] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:23,589] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:23,982] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:25,374] INFO : Plotted jets +rank0 [2026-10-28 13:19:27,051] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:27,084] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:27,294] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:27,659] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:29,033] INFO : Plotted jets +rank0 [2026-10-28 13:19:30,097] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:41,534] INFO : Step 47100/50000 rank0 | Train Loss: 7.8766 | Train Loss Unweighted: 2.8719 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.5630 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1355 | Regression_eta: 0.0101 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6588 | Classification_binary: 2.5630 | Regression_cos_phi: 0.5587 | Regression_energy: 1.4811 | Regression_eta: 0.4494 | Regression_pt: 1.5535 | Regression_sin_phi: 0.6121 | LR: 1.07e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1224 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3242 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6321.9 | uss_MB: 5851.5 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:19:41,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:52,588] INFO : Step 47200/50000 rank0 | Train Loss: 8.0834 | Train Loss Unweighted: 2.8372 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5106 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1445 | Regression_eta: 0.0145 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6882 | Classification_binary: 2.5106 | Regression_cos_phi: 0.3891 | Regression_energy: 1.5796 | Regression_eta: 0.6453 | Regression_pt: 1.6229 | Regression_sin_phi: 0.6477 | LR: 1.07e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4661s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0095 | max_backward: 0.0330 | max_optimizer_step: 0.1414 | max_tensorboard_logging: 0.3801 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6321.9 | uss_MB: 5851.5 | child_rss_MB: 3196.4 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:19:52,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:03,782] INFO : Step 47300/50000 rank0 | Train Loss: 8.7199 | Train Loss Unweighted: 2.8333 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4913 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1458 | Regression_eta: 0.0235 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7360 | Classification_binary: 2.4913 | Regression_cos_phi: 0.4967 | Regression_energy: 1.5940 | Regression_eta: 1.0479 | Regression_pt: 1.6402 | Regression_sin_phi: 0.7138 | LR: 1.06e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4249s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0377 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.3201 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6321.9 | uss_MB: 5851.6 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:03,997] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:14,988] INFO : Step 47400/50000 rank0 | Train Loss: 7.6852 | Train Loss Unweighted: 2.7405 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4170 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1445 | Regression_eta: 0.0085 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6719 | Classification_binary: 2.4170 | Regression_cos_phi: 0.3571 | Regression_energy: 1.5798 | Regression_eta: 0.3777 | Regression_pt: 1.6727 | Regression_sin_phi: 0.6089 | LR: 1.06e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4519s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0903 | max_forward: 0.0169 | max_loss: 0.0036 | max_backward: 0.0382 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6321.9 | uss_MB: 5853.6 | child_rss_MB: 3193.0 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:20:15,209] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:26,093] INFO : Step 47500/50000 rank0 | Train Loss: 8.3020 | Train Loss Unweighted: 2.9092 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.5762 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1459 | Regression_eta: 0.0105 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6603 | Classification_binary: 2.5762 | Regression_cos_phi: 0.4747 | Regression_energy: 1.5955 | Regression_eta: 0.4689 | Regression_pt: 1.6988 | Regression_sin_phi: 0.8276 | LR: 1.06e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0295 | max_forward: 0.0141 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3108 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6203.7 | uss_MB: 5733.0 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:26,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:37,162] INFO : Step 47600/50000 rank0 | Train Loss: 7.6789 | Train Loss Unweighted: 2.7988 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4885 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1358 | Regression_eta: 0.0082 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6744 | Classification_binary: 2.4885 | Regression_cos_phi: 0.4495 | Regression_energy: 1.4850 | Regression_eta: 0.3673 | Regression_pt: 1.6033 | Regression_sin_phi: 0.6110 | LR: 1.05e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4323s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0913 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6204.7 | uss_MB: 5733.9 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:37,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:48,227] INFO : Step 47700/50000 rank0 | Train Loss: 8.0356 | Train Loss Unweighted: 2.7686 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.4499 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1422 | Regression_eta: 0.0170 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6506 | Classification_binary: 2.4499 | Regression_cos_phi: 0.4911 | Regression_energy: 1.5549 | Regression_eta: 0.7572 | Regression_pt: 1.5236 | Regression_sin_phi: 0.6084 | LR: 1.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3145s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0036 | max_backward: 0.0333 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.2677 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6204.7 | uss_MB: 5733.9 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:20:48,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:59,342] INFO : Step 47800/50000 rank0 | Train Loss: 8.5156 | Train Loss Unweighted: 2.8458 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5181 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1402 | Regression_eta: 0.0257 | Regression_pt: 0.1299 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6962 | Classification_binary: 2.5181 | Regression_cos_phi: 0.4170 | Regression_energy: 1.5328 | Regression_eta: 1.1460 | Regression_pt: 1.5434 | Regression_sin_phi: 0.6621 | LR: 1.04e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0982 | max_forward: 0.0148 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6205.7 | uss_MB: 5734.4 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:20:59,559] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:10,373] INFO : Step 47900/50000 rank0 | Train Loss: 7.7110 | Train Loss Unweighted: 2.7371 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4359 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1301 | Regression_eta: 0.0129 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6803 | Classification_binary: 2.4359 | Regression_cos_phi: 0.4427 | Regression_energy: 1.4221 | Regression_eta: 0.5769 | Regression_pt: 1.5012 | Regression_sin_phi: 0.6518 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3724s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.3012 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6206.7 | uss_MB: 5734.7 | child_rss_MB: 3194.8 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:21:10,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:21,463] INFO : Step 48000/50000 rank0 | Train Loss: 7.4383 | Train Loss Unweighted: 2.7485 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4632 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1235 | Regression_eta: 0.0099 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7051 | Classification_binary: 2.4632 | Regression_cos_phi: 0.4259 | Regression_energy: 1.3502 | Regression_eta: 0.4414 | Regression_pt: 1.4264 | Regression_sin_phi: 0.6261 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1059 | max_loss: 0.0035 | max_backward: 0.0345 | max_optimizer_step: 0.1414 | max_tensorboard_logging: 0.3000 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 6206.7 | uss_MB: 5735.2 | child_rss_MB: 3193.7 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:21,675] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:21,730] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 13:21:21,814] INFO : Running validation on rank0 +rank0 [2026-10-28 13:21:21,814] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:21:21,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:21:22,908] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 13:21:23,055] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:21:23,188] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:21:23,263] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:21:23,333] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:21:23,401] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:21:23,470] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:21:23,541] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:21:23,652] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:21:23,786] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:21:23,919] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:21:23,991] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:21:24,061] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:21:24,130] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:21:24,200] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:21:24,270] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:21:24,389] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:21:24,397] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:21:24,400] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:21:24,402] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:21:24,403] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:21:24,406] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:21:24,409] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:21:24,412] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:21:24,415] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:21:24,416] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:21:24,419] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:21:24,421] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:21:24,422] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:21:24,425] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:21:24,426] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:21:24,428] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:21:30,246] INFO : VALIDATION | Step=48000/50000 | Train Loss=7.4383 | Valid Loss=7.8314 | Train Loss Unweighted=2.7485 | Valid Loss Unweighted=2.8333 | Stale=0 | ETA=5.4m +rank0 [2026-10-28 13:21:30,255] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:21:30,312] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:21:30,316] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:21:30,499] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:21:32,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:21:32,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:21:33,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:21:34,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:21:34,355] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:21:34,358] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:21:34,414] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:21:34,417] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:21:34,542] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:21:36,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:21:36,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:21:37,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:21:37,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:21:38,096] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:21:38,100] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:21:38,152] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:21:38,155] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:21:38,280] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:21:39,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:21:40,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:21:41,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:21:41,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:21:41,831] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:21:41,839] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:21:42,101] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:21:42,511] INFO : Plotted number of elements +rank0 [2026-10-28 13:21:43,954] INFO : Plotted jets +rank0 [2026-10-28 13:21:45,051] INFO : Plotted jet ratio +rank0 [2026-10-28 13:21:45,083] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:21:45,318] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:21:45,712] INFO : Plotted number of elements +rank0 [2026-10-28 13:21:47,689] INFO : Plotted jets +rank0 [2026-10-28 13:21:48,804] INFO : Plotted jet ratio +rank0 [2026-10-28 13:21:48,837] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 13:21:49,048] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 13:21:49,411] INFO : Plotted number of elements +rank0 [2026-10-28 13:21:50,799] INFO : Plotted jets +rank0 [2026-10-28 13:21:51,864] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:03,362] INFO : Step 48100/50000 rank0 | Train Loss: 7.9457 | Train Loss Unweighted: 2.8769 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5763 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1304 | Regression_eta: 0.0133 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6980 | Classification_binary: 2.5763 | Regression_cos_phi: 0.5065 | Regression_energy: 1.4254 | Regression_eta: 0.5923 | Regression_pt: 1.4655 | Regression_sin_phi: 0.6817 | LR: 1.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4609s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1072 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0386 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2730 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6396.8 | uss_MB: 5926.1 | child_rss_MB: 3192.2 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:22:03,579] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:14,433] INFO : Step 48200/50000 rank0 | Train Loss: 7.8010 | Train Loss Unweighted: 2.7834 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4778 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1337 | Regression_eta: 0.0112 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6997 | Classification_binary: 2.4778 | Regression_cos_phi: 0.4613 | Regression_energy: 1.4612 | Regression_eta: 0.5008 | Regression_pt: 1.5193 | Regression_sin_phi: 0.6809 | LR: 1.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3733s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0166 | max_forward: 0.0133 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6396.8 | uss_MB: 5924.0 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:14,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:25,424] INFO : Step 48300/50000 rank0 | Train Loss: 7.9939 | Train Loss Unweighted: 2.8113 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4856 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1428 | Regression_eta: 0.0102 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7513 | Classification_binary: 2.4856 | Regression_cos_phi: 0.4222 | Regression_energy: 1.5609 | Regression_eta: 0.4553 | Regression_pt: 1.6545 | Regression_sin_phi: 0.6642 | LR: 1.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4352s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0612 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.3156 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6396.8 | uss_MB: 5924.0 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:22:25,636] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:36,624] INFO : Step 48400/50000 rank0 | Train Loss: 8.5333 | Train Loss Unweighted: 2.9045 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5579 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1531 | Regression_eta: 0.0096 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.8004 | Classification_binary: 2.5579 | Regression_cos_phi: 0.6014 | Regression_energy: 1.6742 | Regression_eta: 0.4276 | Regression_pt: 1.7294 | Regression_sin_phi: 0.7425 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4287s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0755 | max_forward: 0.0150 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6396.8 | uss_MB: 5924.0 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:36,837] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:47,584] INFO : Step 48500/50000 rank0 | Train Loss: 7.7259 | Train Loss Unweighted: 2.8036 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5018 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1337 | Regression_eta: 0.0140 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6815 | Classification_binary: 2.5018 | Regression_cos_phi: 0.4428 | Regression_energy: 1.4620 | Regression_eta: 0.6243 | Regression_pt: 1.4654 | Regression_sin_phi: 0.5480 | LR: 1.02e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0133 | max_loss: 0.0354 | max_backward: 0.0330 | max_optimizer_step: 0.1403 | max_tensorboard_logging: 0.3999 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6278.9 | uss_MB: 5806.0 | child_rss_MB: 3194.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:22:47,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:58,744] INFO : Step 48600/50000 rank0 | Train Loss: 7.5326 | Train Loss Unweighted: 2.7159 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.4124 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1334 | Regression_eta: 0.0083 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7980 | Classification_binary: 2.4124 | Regression_cos_phi: 0.4076 | Regression_energy: 1.4582 | Regression_eta: 0.3683 | Regression_pt: 1.5268 | Regression_sin_phi: 0.5614 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4455s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0839 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3287 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6279.9 | uss_MB: 5807.0 | child_rss_MB: 3193.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:58,957] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:09,898] INFO : Step 48700/50000 rank0 | Train Loss: 8.0103 | Train Loss Unweighted: 2.8529 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5429 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1360 | Regression_eta: 0.0136 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7146 | Classification_binary: 2.5429 | Regression_cos_phi: 0.4032 | Regression_energy: 1.4867 | Regression_eta: 0.6062 | Regression_pt: 1.5135 | Regression_sin_phi: 0.7432 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4631s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0582 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3601 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6280.9 | uss_MB: 5807.2 | child_rss_MB: 3191.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:10,113] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:20,898] INFO : Step 48800/50000 rank0 | Train Loss: 7.3196 | Train Loss Unweighted: 2.6804 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3973 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1236 | Regression_eta: 0.0074 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6393 | Classification_binary: 2.3973 | Regression_cos_phi: 0.4885 | Regression_energy: 1.3513 | Regression_eta: 0.3319 | Regression_pt: 1.4299 | Regression_sin_phi: 0.6814 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4574s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0161 | max_loss: 0.0034 | max_backward: 0.0384 | max_optimizer_step: 0.1415 | max_tensorboard_logging: 0.3797 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6280.9 | uss_MB: 5809.9 | child_rss_MB: 3196.5 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:23:21,111] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:32,043] INFO : Step 48900/50000 rank0 | Train Loss: 7.8801 | Train Loss Unweighted: 2.8892 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.5804 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1369 | Regression_eta: 0.0105 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7035 | Classification_binary: 2.5804 | Regression_cos_phi: 0.5000 | Regression_energy: 1.4961 | Regression_eta: 0.4661 | Regression_pt: 1.5297 | Regression_sin_phi: 0.6043 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.3010 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 6280.9 | uss_MB: 5807.9 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:32,256] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:43,151] INFO : Step 49000/50000 rank0 | Train Loss: 7.4365 | Train Loss Unweighted: 2.7296 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4395 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1267 | Regression_eta: 0.0102 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6775 | Classification_binary: 2.4395 | Regression_cos_phi: 0.4473 | Regression_energy: 1.3851 | Regression_eta: 0.4551 | Regression_pt: 1.4518 | Regression_sin_phi: 0.5801 | LR: 1.01e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1104 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3170 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6280.9 | uss_MB: 5807.9 | child_rss_MB: 3193.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:43,365] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:43,398] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 13:23:43,481] INFO : Running validation on rank0 +rank0 [2026-10-28 13:23:43,481] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:23:43,481] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:23:44,598] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 13:23:44,745] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:23:44,879] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:23:44,954] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:23:45,024] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:23:45,092] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:23:45,162] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:23:45,232] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:23:45,343] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:23:45,477] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:23:45,610] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:23:45,682] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:23:45,751] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:23:45,820] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:23:45,890] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:23:45,960] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:23:46,076] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:23:46,084] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:23:46,086] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:23:46,192] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:23:46,195] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:23:46,198] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:23:46,201] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:23:46,202] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:23:46,204] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:23:46,207] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:23:46,208] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:23:46,210] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:23:46,213] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:23:46,214] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:23:46,216] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:23:46,219] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:23:52,175] INFO : VALIDATION | Step=49000/50000 | Train Loss=7.4365 | Valid Loss=7.8849 | Train Loss Unweighted=2.7296 | Valid Loss Unweighted=2.8501 | Stale=1 | ETA=2.7m +rank0 [2026-10-28 13:23:52,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:23:52,241] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:23:52,244] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:23:52,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:23:54,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:23:54,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:23:55,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:23:55,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:23:56,032] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:23:56,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:23:56,090] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:23:56,093] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:23:56,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:23:57,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:23:58,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:23:59,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:23:59,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:23:59,850] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:23:59,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:23:59,908] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:23:59,911] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:24:00,041] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:24:01,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:24:02,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:24:02,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:24:03,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:24:03,301] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:24:03,309] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:24:03,571] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:24:03,984] INFO : Plotted number of elements +rank0 [2026-10-28 13:24:05,435] INFO : Plotted jets +rank0 [2026-10-28 13:24:06,522] INFO : Plotted jet ratio +rank0 [2026-10-28 13:24:06,554] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:24:06,788] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:24:07,183] INFO : Plotted number of elements +rank0 [2026-10-28 13:24:08,571] INFO : Plotted jets +rank0 [2026-10-28 13:24:10,256] INFO : Plotted jet ratio +rank0 [2026-10-28 13:24:10,289] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 13:24:10,498] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 13:24:10,860] INFO : Plotted number of elements +rank0 [2026-10-28 13:24:12,242] INFO : Plotted jets +rank0 [2026-10-28 13:24:13,310] INFO : Plotted jet ratio +rank0 [2026-10-28 13:24:24,769] INFO : Step 49100/50000 rank0 | Train Loss: 7.7791 | Train Loss Unweighted: 2.8700 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5803 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1252 | Regression_eta: 0.0138 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6637 | Classification_binary: 2.5803 | Regression_cos_phi: 0.4740 | Regression_energy: 1.3686 | Regression_eta: 0.6141 | Regression_pt: 1.4115 | Regression_sin_phi: 0.6669 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5036s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1172 | max_loss: 0.0036 | max_backward: 0.0334 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3029 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6467.5 | uss_MB: 5997.7 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:24:24,981] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 114.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:35,929] INFO : Step 49200/50000 rank0 | Train Loss: 7.6447 | Train Loss Unweighted: 2.7896 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4989 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1273 | Regression_eta: 0.0119 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6280 | Classification_binary: 2.4989 | Regression_cos_phi: 0.5413 | Regression_energy: 1.3921 | Regression_eta: 0.5303 | Regression_pt: 1.4238 | Regression_sin_phi: 0.6303 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0541 | max_forward: 0.0141 | max_loss: 0.0036 | max_backward: 0.0330 | max_optimizer_step: 0.1411 | max_tensorboard_logging: 0.2631 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6467.5 | uss_MB: 5997.7 | child_rss_MB: 3186.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:24:36,145] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:47,051] INFO : Step 49300/50000 rank0 | Train Loss: 7.2660 | Train Loss Unweighted: 2.6454 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3588 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1251 | Regression_eta: 0.0097 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6358 | Classification_binary: 2.3588 | Regression_cos_phi: 0.4704 | Regression_energy: 1.3680 | Regression_eta: 0.4329 | Regression_pt: 1.4467 | Regression_sin_phi: 0.5534 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3536s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0169 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.2688 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6467.5 | uss_MB: 5997.8 | child_rss_MB: 3185.5 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:47,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 38.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:58,063] INFO : Step 49400/50000 rank0 | Train Loss: 8.1874 | Train Loss Unweighted: 2.9322 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6051 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1438 | Regression_eta: 0.0119 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7466 | Classification_binary: 2.6051 | Regression_cos_phi: 0.4762 | Regression_energy: 1.5725 | Regression_eta: 0.5301 | Regression_pt: 1.6360 | Regression_sin_phi: 0.6210 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1039 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.3042 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6467.5 | uss_MB: 5997.8 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:58,276] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 39.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:09,176] INFO : Step 49500/50000 rank0 | Train Loss: 7.4606 | Train Loss Unweighted: 2.8836 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5962 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1252 | Regression_eta: 0.0089 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.5962 | Regression_cos_phi: 0.4405 | Regression_energy: 1.3686 | Regression_eta: 0.3975 | Regression_pt: 1.4642 | Regression_sin_phi: 0.5237 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4896s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0867 | max_loss: 0.0035 | max_backward: 0.0388 | max_optimizer_step: 0.1441 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6467.5 | uss_MB: 5999.9 | child_rss_MB: 3187.3 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:25:09,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:20,242] INFO : Step 49600/50000 rank0 | Train Loss: 8.0624 | Train Loss Unweighted: 2.8580 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5260 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1493 | Regression_eta: 0.0102 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.7390 | Classification_binary: 2.5260 | Regression_cos_phi: 0.4297 | Regression_energy: 1.6317 | Regression_eta: 0.4530 | Regression_pt: 1.6606 | Regression_sin_phi: 0.6224 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4769s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1004 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3120 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6349.7 | uss_MB: 5879.9 | child_rss_MB: 3187.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:25:20,456] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:31,434] INFO : Step 49700/50000 rank0 | Train Loss: 7.6107 | Train Loss Unweighted: 2.8008 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5044 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1294 | Regression_eta: 0.0099 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6974 | Classification_binary: 2.5044 | Regression_cos_phi: 0.3312 | Regression_energy: 1.4147 | Regression_eta: 0.4419 | Regression_pt: 1.4943 | Regression_sin_phi: 0.7268 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4641s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1051 | max_forward: 0.0141 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1438 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6349.7 | uss_MB: 5880.4 | child_rss_MB: 3187.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:25:31,647] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 39.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:42,442] INFO : Step 49800/50000 rank0 | Train Loss: 8.2056 | Train Loss Unweighted: 2.8163 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4889 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1438 | Regression_eta: 0.0225 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6846 | Classification_binary: 2.4889 | Regression_cos_phi: 0.4349 | Regression_energy: 1.5725 | Regression_eta: 1.0052 | Regression_pt: 1.5591 | Regression_sin_phi: 0.4604 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4444s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0560 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1428 | max_tensorboard_logging: 0.3104 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6349.7 | uss_MB: 5880.5 | child_rss_MB: 3187.0 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:25:42,654] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 52.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 39.0°C 98.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 37.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 40.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:53,593] INFO : Step 49900/50000 rank0 | Train Loss: 7.9661 | Train Loss Unweighted: 2.8305 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4954 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1474 | Regression_eta: 0.0125 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6782 | Classification_binary: 2.4954 | Regression_cos_phi: 0.3538 | Regression_energy: 1.6117 | Regression_eta: 0.5585 | Regression_pt: 1.7350 | Regression_sin_phi: 0.5335 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0438 | max_forward: 0.0161 | max_loss: 0.0037 | max_backward: 0.0380 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.2548 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6349.7 | uss_MB: 5882.7 | child_rss_MB: 3188.9 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:25:53,806] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:04,718] INFO : Step 50000/50000 rank0 | Train Loss: 7.9695 | Train Loss Unweighted: 2.8070 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4928 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1375 | Regression_eta: 0.0136 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 37.6435 | Classification_binary: 1.0000 | Regression_cos_phi: 64.8940 | Regression_energy: 10.9326 | Regression_eta: 44.5986 | Regression_pt: 11.8821 | Regression_sin_phi: 94.6450 | Task Weighted Losses: Classification: 0.6647 | Classification_binary: 2.4928 | Regression_cos_phi: 0.4480 | Regression_energy: 1.5033 | Regression_eta: 0.6055 | Regression_pt: 1.5595 | Regression_sin_phi: 0.6957 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3528s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0035 | max_backward: 0.0342 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.2693 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6349.7 | uss_MB: 5880.7 | child_rss_MB: 3190.6 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:26:04,936] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 117.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 99.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 38.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:04,954] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 13:26:05,039] INFO : Running validation on rank0 +rank0 [2026-10-28 13:26:05,039] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:26:05,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:26:06,141] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 13:26:06,288] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:26:06,422] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:26:06,497] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:26:06,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:26:06,634] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:26:06,704] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:26:06,775] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:26:06,886] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:26:07,019] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:26:07,152] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:26:07,223] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:26:07,294] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:26:07,362] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:26:07,432] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:26:07,502] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:26:07,619] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:26:07,700] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:26:07,703] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:26:07,707] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:26:07,709] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:26:07,712] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:26:07,713] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:26:07,715] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:26:07,718] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:26:07,720] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:26:07,723] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:26:07,726] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:26:07,728] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:26:07,731] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:26:07,734] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:26:07,735] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:26:13,675] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.9695 | Valid Loss=7.8414 | Train Loss Unweighted=2.8070 | Valid Loss Unweighted=2.8376 | Stale=2 | ETA=0.0m +rank0 [2026-10-28 13:26:13,685] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:13,743] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:26:13,747] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:13,935] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:15,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:16,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:16,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:17,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:17,596] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:26:17,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:17,654] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:26:17,657] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:17,786] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:19,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:20,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:20,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:21,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:21,344] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:26:21,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:21,401] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:26:21,404] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:26:21,541] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:23,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:23,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:24,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:24,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:24,876] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:26:24,883] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:25,147] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:25,561] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:27,004] INFO : Plotted jets +rank0 [2026-10-28 13:26:28,095] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:28,127] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:28,362] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:28,755] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:30,143] INFO : Plotted jets +rank0 [2026-10-28 13:26:31,807] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:31,840] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:32,051] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_attention_seed12345_20261028_111201_967206/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:32,415] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:33,788] INFO : Plotted jets +rank0 [2026-10-28 13:26:34,850] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:35,316] INFO : Training completed. Total time on device 0: 134.032min diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-50000.pth b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..2a02d897c87d56013bb79f1dfaefaaa7f3148216 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01e13a81252b32cd0ba43135e00c12b229aee6e8a9951af527a956d65e947b2a +size 103962061 diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_1000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..719793d03baecb1fe2496b3ebf5d0da4515eb930 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.0473664266767715, "train": {"Classification_binary": 4.216756820678711, "Classification": 0.03874162584543228, "Regression_pt": 0.23588064312934875, "Regression_eta": 0.019524268805980682, "Regression_sin_phi": 0.012373658828437328, "Regression_cos_phi": 0.012103606015443802, "Regression_energy": 0.2517673075199127, "Total": 14.195218086242676, "Total_unweighted": 4.7871479308232665}, "train_group": {}, "valid": {"Classification_binary": 4.136436780293782, "Classification": 0.034493779142697654, "Regression_pt": 0.24985798199971518, "Regression_eta": 0.017602877070506413, "Regression_sin_phi": 0.009527412553628286, "Regression_cos_phi": 0.010284734889864922, "Regression_energy": 0.25848660866419476, "Total": 13.756724039713541, "Total_unweighted": 4.716690174614389, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.03468842804431915, "diagnostic/loss/cld_pf/Classification_binary": 4.11064338684082, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.009902775287628174, "diagnostic/loss/cld_pf/Regression_energy": 0.26089441776275635, "diagnostic/loss/cld_pf/Regression_eta": 0.01647782512009144, "diagnostic/loss/cld_pf/Regression_pt": 0.2538081407546997, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.008854411542415619, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0575791671872139, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.24495366215705872, "diagnostic/regression/cld_pf/energy_residual_mean": 0.030459890142083168, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5223631858825684, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.029040297493338585, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2389049530029297, "diagnostic/regression/cld_pf/pt_residual_mean": 0.05620743706822395, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5182451605796814, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9058415293693542, "metrics/particle/class_1/pid_accuracy": 0.9814239144325256, "metrics/particle/class_2/efficiency": 0.19687820971012115, "metrics/particle/class_2/pid_accuracy": 0.30497685074806213, "metrics/particle/class_3/efficiency": 0.6685126423835754, "metrics/particle/class_3/pid_accuracy": 0.9713346362113953, "metrics/particle/class_4/efficiency": 0.8980827331542969, "metrics/particle/class_4/pid_accuracy": 0.020224718376994133, "metrics/particle/class_5/efficiency": 0.968932032585144, "metrics/particle/class_5/pid_accuracy": 0.1442885845899582, "metrics/particle/count/bias_mean": -0.8248366117477417, "metrics/particle/count/mae": 5.181699275970459, "metrics/particle/count/prediction_mean": 69.74052429199219, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.10281331837177277, "metrics/particle/event/energy_response_mean": 0.9604195952415466, "metrics/particle/event/met_abs_error": 11.532632827758789, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09087158739566803, "metrics/particle/event/scalar_pt_response_mean": 0.974582314491272, "metrics/particle/event/vector_pt_closure": 0.09244699031114578, "metrics/particle/matched/delta_eta_abs_mean": 0.026945412158966064, "metrics/particle/matched/delta_phi_abs_mean": 0.015329387038946152, "metrics/particle/matched/delta_r_mean": 0.0335933081805706, "metrics/particle/matched/energy_relative_abs_mean": 0.09646997600793839, "metrics/particle/matched/pid_accuracy": 0.94740229845047, "metrics/particle/matched/pt_relative_abs_mean": 0.08809234946966171, "metrics/particle/matching/duplicate_fraction": 0.04783370718359947, "metrics/particle/matching/efficiency": 0.7484092116355896, "metrics/particle/matching/f1": 0.7528089880943298, "metrics/particle/matching/purity": 0.757260799407959, "metrics/particle/matching/target_coverage_dr0p05": 0.6248599290847778, "metrics/particle/matching/target_coverage_dr0p10": 0.781058669090271, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6649747490882874, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8041772842407227, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03569335490465164, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08921569585800171, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08449803292751312, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005242963321506977, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1358460783958435, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006241519469767809, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4464375972747803, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0917196050286293, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8358441591262817, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7370.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4156518876552582, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5986599922180176, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7942732572555542, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6430559158325195, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16790123283863068, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.823577880859375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.026131687685847282, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.175704836845398, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.14866849780082703, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.230922698974609, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9720.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7270395755767822, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0631183385849, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 15.399791717529297, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.633331298828125, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15704885125160217, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5465432405471802, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 13.406599998474121, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 5.094624042510986, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 23.951499938964844, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04017508774995804, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09970619529485703, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10285992175340652, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.009666926227509975, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15257853269577026, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017332328483462334, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9633731842041016, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0033964430913329124, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2225027084350586, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1327.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.994452178478241, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.076859951019287, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 55370.20703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 55369.98046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.326241135597229, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2789188.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09889677166938782, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.7127013206481934, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0474633164703846, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.182598114013672, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2538.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4827210009098053, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.4939751625061035, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.23095417022705, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7834749221801758, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.3000817894935608, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.03221537917852402, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.97095012664795, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 5.042307376861572, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.996082305908203, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0322980135679245, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10103565454483032, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09102486819028854, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0021770948078483343, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1433052122592926, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00044043161324225366, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.16316716372966766, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.07447348535060883, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5353286862373352, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18164.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2737576365470886, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1105620861053467, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8628524541854858, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7003391981124878, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13636156916618347, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.914036750793457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006813536398112774, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4799747169017792, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.10704055428504944, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.2812633514404297, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22015.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7944714426994324, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -6.60261344909668, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 16.672088623046875, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.5037841796875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.022749185562134, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.4684151411056519, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.260048866271973, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.0525572299957275, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 15.386326789855957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.033618759363889694, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09772687405347824, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08981874585151672, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.003388310084119439, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1417766660451889, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002866609487682581, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.28042176365852356, "metrics/particle/momentum/all/accepted/pz_bias": 0.07535842806100845, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2773194313049316, "metrics/particle/momentum/all/accepted/samples": 26861.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.36736878752708435, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3205459117889404, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4101.36328125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4101.19921875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1593674272298813, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 759010.25, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.019111253321170807, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.916678786277771, "metrics/particle/momentum/all/assigned/pz_bias": 0.114434614777565, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.012198448181152, "metrics/particle/momentum/all/assigned/samples": 34273.0, "metrics/particle/momentum/all/efficiency": 0.7513776421546936, "metrics/particle/momentum/all/event/energy_bias": -10.159707069396973, "metrics/particle/momentum/all/event/p3_error_mean": 29.912471771240234, "metrics/particle/momentum/all/event/pt_vector_error_mean": 16.039064407348633, "metrics/particle/momentum/all/event/px_bias": -2.1657819747924805, "metrics/particle/momentum/all/event/py_bias": -0.8896564245223999, "metrics/particle/momentum/all/event/pz_abs_mean": 21.758522033691406, "metrics/particle/momentum/all/event/pz_bias": 12.189488410949707, "metrics/particle/momentum/all/event/pz_residual_rms": 31.30167579650879, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02580471709370613, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.062333788722753525, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0485362783074379, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.005552720744162798, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09332192689180374, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016803464386612177, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.21252642571926117, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.07615391165018082, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8703628182411194, "metrics/particle/momentum/class_1/accepted/samples": 15473.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14763225615024567, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.41817301511764526, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6733663082122803, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5852528810501099, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07164527475833893, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.113224983215332, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008576429449021816, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5189562439918518, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.08797707408666611, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.145256757736206, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9025841355323792, "metrics/particle/momentum/class_1/event/energy_bias": 10.193073272705078, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.784221649169922, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.657356262207031, "metrics/particle/momentum/class_1/event/px_bias": 0.2904914915561676, "metrics/particle/momentum/class_1/event/py_bias": -1.4390203952789307, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.120099067687988, "metrics/particle/momentum/class_1/event/pz_bias": 1.1514872312545776, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.792207717895508, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05099354311823845, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2292470932006836, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22841522097587585, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07646556943655014, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26549801230430603, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.06591639667749405, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.7188825607299805, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.09296675026416779, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.9866485595703125, "metrics/particle/momentum/class_2/accepted/samples": 622.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.4552875757217407, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.1598643064498901, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 60490.8125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 60490.09765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5510116219520569, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2915406.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.12957382202148438, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.8934197425842285, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.036592815071344376, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.35370922088623, "metrics/particle/momentum/class_2/assigned/samples": 2323.0, "metrics/particle/momentum/class_2/efficiency": 0.21522490680217743, "metrics/particle/momentum/class_2/event/energy_bias": -21.466358184814453, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.030399322509766, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.015254020690918, "metrics/particle/momentum/class_2/event/px_bias": -1.0568574666976929, "metrics/particle/momentum/class_2/event/py_bias": 1.14505136013031, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.668736457824707, "metrics/particle/momentum/class_2/event/pz_bias": 2.841106414794922, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.932273864746094, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0445297472178936, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14438006281852722, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1446428745985031, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.022335318848490715, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18573465943336487, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008756567258387804, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2246495932340622, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.07404465973377228, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9371128082275391, "metrics/particle/momentum/class_3/accepted/samples": 10278.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4630517065525055, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4427573680877686, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.382337808609009, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.21303129196167, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20471890270709991, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.915136337280273, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01421270053833723, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7564644813537598, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.160831019282341, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.016674757003784, "metrics/particle/momentum/class_3/assigned/samples": 14283.0, "metrics/particle/momentum/class_3/efficiency": 0.6766738891601562, "metrics/particle/momentum/class_3/event/energy_bias": 8.251482009887695, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.289270401000977, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.617317199707031, "metrics/particle/momentum/class_3/event/px_bias": -1.414245843887329, "metrics/particle/momentum/class_3/event/py_bias": -0.616672158241272, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.835617065429688, "metrics/particle/momentum/class_3/event/pz_bias": 7.386456489562988, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.999271392822266, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03001575358211994, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.08496332168579102, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.08452421426773071, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.008706287480890751, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1344994455575943, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003144653979688883, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5057897567749023, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.24642646312713623, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4215521812438965, "metrics/particle/momentum/class_4/accepted/samples": 318.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.16113246977329254, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3723971843719482, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.7791773676872253, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.6674831509590149, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0762711837887764, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.069228172302246, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7470571398735046, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.38059303164482117, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.9147515296936035, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8983050584793091, "metrics/particle/momentum/class_4/event/energy_bias": -5.081299781799316, "metrics/particle/momentum/class_4/event/p3_error_mean": 6.052696704864502, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.791170120239258, "metrics/particle/momentum/class_4/event/px_bias": 0.2429051399230957, "metrics/particle/momentum/class_4/event/py_bias": 0.3244225084781647, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.0383942127227783, "metrics/particle/momentum/class_4/event/pz_bias": 0.8387882113456726, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.32275390625, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.028337441384792328, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.041190654039382935, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03545228764414787, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.013538720086216927, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07700250297784805, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.48855140805244446, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.37825828790664673, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9159852266311646, "metrics/particle/momentum/class_5/accepted/samples": 170.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.051859911531209946, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.12422467768192291, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.11939916759729385, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05268699303269386, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9968525767326355, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.49691393971443176, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.38853344321250916, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9202417731285095, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9826589822769165, "metrics/particle/momentum/class_5/event/energy_bias": -2.0566046237945557, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.741605758666992, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.954216957092285, "metrics/particle/momentum/class_5/event/px_bias": -0.22807539999485016, "metrics/particle/momentum/class_5/event/py_bias": -0.30343788862228394, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.8368875980377197, "metrics/particle/momentum/class_5/event/pz_bias": -0.028350383043289185, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.642534255981445, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.3158166408538818, "step_valid_time": 451.3071744441986, "step_total_time": 452.6229918003082, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9760626214900792, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19716495547670154, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3032, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2725, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.894322284213981, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8987467018469657, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8965290343806547, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1012532981530343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8618313094847391, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8660949868073878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8639578878104951, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.894322284213981, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.968014697278444, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19424947235273937, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2262, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1997, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1953, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8766461808604039, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8828470380194519, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8797356828193833, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11715296198054818, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8573309920983319, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.863395225464191, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.860352422907489, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8766461808604039, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.97528501255521, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20310234302187835, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1295, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1095, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1060, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.830804248861912, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8455598455598455, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8381171067738232, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15444015444015444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8042488619119879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8185328185328186, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.81132797550708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.830804248861912} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_10000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..5281f2fb0f52d2a800e639ded741608db41a0303 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47366426676771506, "train": {"Classification_binary": 2.927682876586914, "Classification": 0.021192453801631927, "Regression_pt": 0.17359738051891327, "Regression_eta": 0.010025660507380962, "Regression_sin_phi": 0.006973402574658394, "Regression_cos_phi": 0.008110662922263145, "Regression_energy": 0.1790442019701004, "Total": 9.45895004272461, "Total_unweighted": 3.326626638881862}, "train_group": {}, "valid": {"Classification_binary": 2.901821772257487, "Classification": 0.02113717297712962, "Regression_pt": 0.15672636032104492, "Regression_eta": 0.012776471674442291, "Regression_sin_phi": 0.007247040048241615, "Regression_cos_phi": 0.007766737913091977, "Regression_energy": 0.16221665342648825, "Total": 9.172136306762695, "Total_unweighted": 3.2696922086179256, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.02129700593650341, "diagnostic/loss/cld_pf/Classification_binary": 2.890777826309204, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007305091246962547, "diagnostic/loss/cld_pf/Regression_energy": 0.16077947616577148, "diagnostic/loss/cld_pf/Regression_eta": 0.011711148545145988, "diagnostic/loss/cld_pf/Regression_pt": 0.15682588517665863, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006738693453371525, "diagnostic/regression/cld_pf/energy_pred_mean": 0.10323843359947205, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21163278818130493, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07611915469169617, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4508759677410126, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05212805047631264, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20554623007774353, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07929519563913345, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4453718960285187, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9286097884178162, "metrics/particle/class_1/pid_accuracy": 0.9840050935745239, "metrics/particle/class_2/efficiency": 0.39113593101501465, "metrics/particle/class_2/pid_accuracy": 0.7075444459915161, "metrics/particle/class_3/efficiency": 0.750109851360321, "metrics/particle/class_3/pid_accuracy": 0.9818357825279236, "metrics/particle/class_4/efficiency": 0.8203834295272827, "metrics/particle/class_4/pid_accuracy": 0.2460024654865265, "metrics/particle/class_5/efficiency": 0.9650485515594482, "metrics/particle/class_5/pid_accuracy": 0.5211267471313477, "metrics/particle/count/bias_mean": -1.5732026100158691, "metrics/particle/count/mae": 3.724836587905884, "metrics/particle/count/prediction_mean": 68.99215698242188, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05916719511151314, "metrics/particle/event/energy_response_mean": 0.9894292950630188, "metrics/particle/event/met_abs_error": 6.829526424407959, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04893499240279198, "metrics/particle/event/scalar_pt_response_mean": 1.004831314086914, "metrics/particle/event/vector_pt_closure": 0.05435725674033165, "metrics/particle/matched/delta_eta_abs_mean": 0.011775044724345207, "metrics/particle/matched/delta_phi_abs_mean": 0.012444956228137016, "metrics/particle/matched/delta_r_mean": 0.01919586770236492, "metrics/particle/matched/energy_relative_abs_mean": 0.09476031363010406, "metrics/particle/matched/pid_accuracy": 0.9627844095230103, "metrics/particle/matched/pt_relative_abs_mean": 0.08744096755981445, "metrics/particle/matching/duplicate_fraction": 0.03571496158838272, "metrics/particle/matching/efficiency": 0.8088639974594116, "metrics/particle/matching/f1": 0.8179821372032166, "metrics/particle/matching/purity": 0.8273082375526428, "metrics/particle/matching/target_coverage_dr0p05": 0.7673783302307129, "metrics/particle/matching/target_coverage_dr0p10": 0.8442736268043518, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7042837738990784, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8465243577957153, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021836692467331886, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08956651389598846, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08500026166439056, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021264873445034027, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.137051060795784, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005629925988614559, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3878990709781647, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.007221618667244911, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.653910756111145, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7993.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29254046082496643, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1721967458724976, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.173275351524353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.062700629234314, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1298205852508545, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.201225280761719, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016910703852772713, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8208390474319458, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04618452861905098, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.002084255218506, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9698.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7884975671768188, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.344139814376831, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.32588005065918, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5182652473449707, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.021493706852197647, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.411427766084671, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.293810844421387, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.35963964462280273, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.180122375488281, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030728867277503014, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09630312770605087, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09683706611394882, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01888255774974823, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1500806361436844, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02302631549537182, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9994296431541443, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17835633456707, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.280855894088745, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1520.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7731444239616394, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6858004331588745, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17342.68359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17342.515625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2636655867099762, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 864897.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07918006181716919, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.932724714279175, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.16718091070652008, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.179513931274414, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2488.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5529283285140991, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.1167831420898438, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.37210750579834, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0579352378845215, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03830132633447647, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1129252091050148, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.218542098999023, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6291109323501587, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.543124198913574, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01708770915865898, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09936247766017914, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08997367322444916, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019452936947345734, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14427699148654938, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005141388392075896, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11698614060878754, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.014221993274986744, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.504360556602478, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19450.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1661660224199295, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7813575863838196, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5943982005119324, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4802584946155548, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10060551762580872, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.057109832763672, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0031845704652369022, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.29763278365135193, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02953493781387806, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9665756225585938, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22295.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8507195115089417, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.1958348751068115, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.618087768554688, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.117810249328613, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8420546054840088, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.4393191933631897, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.032336235046387, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.805945873260498, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.7822747230529785, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019114196300506592, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.096498504281044, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08896134048700333, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019923048093914986, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14263412356376648, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00310741295106709, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2380620390176773, "metrics/particle/momentum/all/accepted/pz_bias": 0.0021834257058799267, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2209148406982422, "metrics/particle/momentum/all/accepted/samples": 28963.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24550655484199524, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9565442204475403, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1252.0875244140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1251.9705810546875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12058815360069275, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 232327.109375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012528639286756516, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6349247097969055, "metrics/particle/momentum/all/assigned/pz_bias": 0.02002357505261898, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1405394077301025, "metrics/particle/momentum/all/assigned/samples": 34481.0, "metrics/particle/momentum/all/efficiency": 0.8101764917373657, "metrics/particle/momentum/all/event/energy_bias": -1.2650880813598633, "metrics/particle/momentum/all/event/p3_error_mean": 17.2204647064209, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.102705001831055, "metrics/particle/momentum/all/event/px_bias": 0.8252469301223755, "metrics/particle/momentum/all/event/py_bias": -0.9636721611022949, "metrics/particle/momentum/all/event/pz_abs_mean": 12.577705383300781, "metrics/particle/momentum/all/event/pz_bias": 1.794696569442749, "metrics/particle/momentum/all/event/pz_residual_rms": 19.204992294311523, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009961341507732868, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.059006381779909134, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04643409326672554, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.018471233546733856, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09576983749866486, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016982200322672725, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1502150595188141, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0029228462371975183, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9065144658088684, "metrics/particle/momentum/class_1/accepted/samples": 15899.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09185842424631119, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30247169733047485, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30316081643104553, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2485790103673935, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05280662700533867, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.25223970413208, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005659936927258968, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3253965675830841, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.003947040997445583, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.8980584144592285, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.927433967590332, "metrics/particle/momentum/class_1/event/energy_bias": 5.342954635620117, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.451079368591309, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.162240982055664, "metrics/particle/momentum/class_1/event/px_bias": -0.005119778215885162, "metrics/particle/momentum/class_1/event/py_bias": -0.663057267665863, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.855302810668945, "metrics/particle/momentum/class_1/event/pz_bias": 0.04197477549314499, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.74354362487793, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03986792266368866, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20286795496940613, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20453695952892303, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04572773352265358, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24405202269554138, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0486111119389534, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.973543405532837, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.22499728202819824, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.601129531860352, "metrics/particle/momentum/class_2/accepted/samples": 1152.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9391993880271912, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8186256289482117, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17488.435546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17487.9609375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.379813551902771, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 868570.8125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08715038746595383, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.457412004470825, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1651081144809723, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.31450366973877, "metrics/particle/momentum/class_2/assigned/samples": 2467.0, "metrics/particle/momentum/class_2/efficiency": 0.39861592650413513, "metrics/particle/momentum/class_2/event/energy_bias": -9.56182861328125, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.50041389465332, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.92995548248291, "metrics/particle/momentum/class_2/event/px_bias": -0.18855448067188263, "metrics/particle/momentum/class_2/event/py_bias": -0.07344324886798859, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.364937782287598, "metrics/particle/momentum/class_2/event/pz_bias": -1.3328557014465332, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.07575035095215, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030016105622053146, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1363580971956253, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1363845318555832, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02761506661772728, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1784440577030182, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005235145217739046, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18057073652744293, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.030164994299411774, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6688651442527771, "metrics/particle/momentum/class_3/accepted/samples": 11461.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3121199607849121, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6647496223449707, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5559297800064087, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4237350225448608, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15693379938602448, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.511849403381348, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008083623833954334, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5197687149047852, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07277712225914001, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6418752670288086, "metrics/particle/momentum/class_3/assigned/samples": 14350.0, "metrics/particle/momentum/class_3/efficiency": 0.7545592188835144, "metrics/particle/momentum/class_3/event/energy_bias": 4.9704694747924805, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.576675415039062, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.98795747756958, "metrics/particle/momentum/class_3/event/px_bias": 0.7575229406356812, "metrics/particle/momentum/class_3/event/py_bias": -0.24971017241477966, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.447674751281738, "metrics/particle/momentum/class_3/event/pz_bias": 2.8841800689697266, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.694939613342285, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014382457360625267, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19225770235061646, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12202345579862595, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06703630834817886, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17649509012699127, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003496503457427025, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4925064444541931, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04467003792524338, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4017903804779053, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2524617314338684, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.322196006774902, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.881910800933838, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.8266825675964355, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16430595517158508, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 53.536556243896484, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008498583920300007, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8076326847076416, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.12237689644098282, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.576401710510254, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.7440831661224365, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.688721179962158, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0184383392333984, "metrics/particle/momentum/class_4/event/px_bias": 0.10236705839633942, "metrics/particle/momentum/class_4/event/py_bias": 0.04739938676357269, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.441985011100769, "metrics/particle/momentum/class_4/event/pz_bias": 0.1790688931941986, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.837610721588135, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007111105136573315, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03184226527810097, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028504937887191772, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0022231664042919874, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07571697235107422, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13831038773059845, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.06308559328317642, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.42773348093032837, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03466318920254707, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0659208595752716, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05799493566155434, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0012293665204197168, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17265768349170685, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.24845072627067566, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1673842817544937, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.199347972869873, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": -0.27260035276412964, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0447269678115845, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8779659271240234, "metrics/particle/momentum/class_5/event/px_bias": 0.15903125703334808, "metrics/particle/momentum/class_5/event/py_bias": -0.024860866367816925, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.44248104095458984, "metrics/particle/momentum/class_5/event/pz_bias": 0.02232816070318222, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7176544666290283, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6998507976531982, "step_valid_time": 4448.551266908646, "step_total_time": 4450.251117944717, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005547253892134, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09662502068223144, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9458483754512635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9480263157894737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9469360933136192, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05197368421052632, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9392845421726288, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9414473684210526, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9403647116806308, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9458483754512635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048929970719072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09484279313382138, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2134, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2125, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9367866549604916, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9396741523557904, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9382281820180259, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0603258476442096, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9328358208955224, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9357111404667547, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9342712684106397, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9367866549604916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042195309800952, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09802257971012429, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1214, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9233687405159332, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9240698557327259, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.923719165085389, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07593014426727411, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.921092564491654, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9217919514047077, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9214421252371916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9233687405159332} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_11000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..48ae511e7d672fe528f86893bf1d17f33d1c4ebd --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5210306934444866, "train": {"Classification_binary": 2.893277645111084, "Classification": 0.019909927621483803, "Regression_pt": 0.14432087540626526, "Regression_eta": 0.009905329905450344, "Regression_sin_phi": 0.006545975338667631, "Regression_cos_phi": 0.006249401718378067, "Regression_energy": 0.15107488632202148, "Total": 8.543539047241211, "Total_unweighted": 3.2312840414233506}, "train_group": {}, "valid": {"Classification_binary": 2.833454132080078, "Classification": 0.02035780002673467, "Regression_pt": 0.15226631363232931, "Regression_eta": 0.01232281948129336, "Regression_sin_phi": 0.007114112998048465, "Regression_cos_phi": 0.007563404738903046, "Regression_energy": 0.1577397584915161, "Total": 8.920054117838541, "Total_unweighted": 3.190818341448903, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020414885133504868, "diagnostic/loss/cld_pf/Classification_binary": 2.8307244777679443, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007274714298546314, "diagnostic/loss/cld_pf/Regression_energy": 0.15760891139507294, "diagnostic/loss/cld_pf/Regression_eta": 0.011429540812969208, "diagnostic/loss/cld_pf/Regression_pt": 0.1534675657749176, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006642629392445087, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08184332400560379, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2010977566242218, "diagnostic/regression/cld_pf/energy_residual_mean": 0.054724037647247314, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4446043074131012, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03124779462814331, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1969536393880844, "diagnostic/regression/cld_pf/pt_residual_mean": 0.05841493234038353, "diagnostic/regression/cld_pf/pt_residual_rms": 0.43794339895248413, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9333818554878235, "metrics/particle/class_1/pid_accuracy": 0.9846002459526062, "metrics/particle/class_2/efficiency": 0.40264326333999634, "metrics/particle/class_2/pid_accuracy": 0.7079796195030212, "metrics/particle/class_3/efficiency": 0.7686576843261719, "metrics/particle/class_3/pid_accuracy": 0.9822168946266174, "metrics/particle/class_4/efficiency": 0.8617557883262634, "metrics/particle/class_4/pid_accuracy": 0.26932084560394287, "metrics/particle/class_5/efficiency": 0.9708737730979919, "metrics/particle/class_5/pid_accuracy": 0.5799999833106995, "metrics/particle/count/bias_mean": -0.6816993355751038, "metrics/particle/count/mae": 3.507843255996704, "metrics/particle/count/prediction_mean": 69.88365936279297, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05683267489075661, "metrics/particle/event/energy_response_mean": 0.9833734631538391, "metrics/particle/event/met_abs_error": 6.127441883087158, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04647304117679596, "metrics/particle/event/scalar_pt_response_mean": 0.9917824864387512, "metrics/particle/event/vector_pt_closure": 0.051829706877470016, "metrics/particle/matched/delta_eta_abs_mean": 0.011125102639198303, "metrics/particle/matched/delta_phi_abs_mean": 0.011594017036259174, "metrics/particle/matched/delta_r_mean": 0.018025321885943413, "metrics/particle/matched/energy_relative_abs_mean": 0.08822651207447052, "metrics/particle/matched/pid_accuracy": 0.9634401202201843, "metrics/particle/matched/pt_relative_abs_mean": 0.0839647501707077, "metrics/particle/matching/duplicate_fraction": 0.03415574133396149, "metrics/particle/matching/efficiency": 0.8203306794166565, "metrics/particle/matching/f1": 0.8243123292922974, "metrics/particle/matching/purity": 0.828332781791687, "metrics/particle/matching/target_coverage_dr0p05": 0.7847635746002197, "metrics/particle/matching/target_coverage_dr0p10": 0.8519612550735474, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7192701101303101, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8585467338562012, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02038986049592495, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08401422947645187, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08072970807552338, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008576782420277596, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1329183727502823, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005781768821179867, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3788035809993744, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03351593390107155, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6116368770599365, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8129.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28665226697921753, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1438363790512085, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1371475458145142, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0165306329727173, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12274257838726044, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.48423957824707, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016324864700436592, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7985430359840393, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.07075812667608261, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.037087917327881, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9801.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8019137978553772, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7058996558189392, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.760814666748047, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3031086921691895, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.042599622160196304, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07455751299858093, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.7735395431518555, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.275315284729004, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.46794605255127, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02859310805797577, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09129947423934937, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09113326668739319, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0036391562316566706, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14267046749591827, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0258397925645113, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9929948449134827, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.23115575313568115, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.367969036102295, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1548.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7597635388374329, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6411893367767334, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6781.81494140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6781.63818359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27419355511665344, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 341772.53125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0857592448592186, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.11147403717041, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.12688057124614716, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.124210357666016, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2542.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5631138682365417, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.9886651039123535, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.625599384307861, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1596705913543701, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.036596499383449554, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.09163463860750198, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.4667840003967285, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.47418421506881714, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.392555236816406, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01625092886388302, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09102068096399307, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08603812009096146, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006425053346902132, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14165258407592773, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006608713301829994, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1144815981388092, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005980849731713533, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5294912457466125, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19671.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16423650085926056, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6457251310348511, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5295155048370361, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4034183919429779, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09408770501613617, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.198936462402344, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034830758813768625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27942436933517456, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.013408271595835686, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.329891324043274, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22394.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8603857755661011, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.5130114555358887, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.783790588378906, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.58071756362915, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.0620390847325325, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.9510467052459717, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.426252841949463, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.21264633536338806, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.371478080749512, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0180483628064394, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08909469097852707, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08483652025461197, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.006874107755720615, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13934296369552612, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0034073872957378626, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23403359949588776, "metrics/particle/momentum/all/accepted/pz_bias": -0.025484854355454445, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2270563840866089, "metrics/particle/momentum/all/accepted/samples": 29348.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24235573410987854, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8591130971908569, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 496.9449768066406, "metrics/particle/momentum/all/assigned/pt_relative_bias": 496.81658935546875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11535250395536423, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 92454.65625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013127213343977928, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6331380605697632, "metrics/particle/momentum/all/assigned/pz_bias": -0.03789318725466728, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.353125810623169, "metrics/particle/momentum/all/assigned/samples": 34737.0, "metrics/particle/momentum/all/efficiency": 0.8209460377693176, "metrics/particle/momentum/all/event/energy_bias": -4.207576274871826, "metrics/particle/momentum/all/event/p3_error_mean": 15.932758331298828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.444859504699707, "metrics/particle/momentum/all/event/px_bias": -0.06804218888282776, "metrics/particle/momentum/all/event/py_bias": -1.9681239128112793, "metrics/particle/momentum/all/event/pz_abs_mean": 11.582833290100098, "metrics/particle/momentum/all/event/pz_bias": -1.9621458053588867, "metrics/particle/momentum/all/event/pz_residual_rms": 19.801870346069336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009027745574712753, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04704218730330467, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04072660580277443, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0063966684974730015, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09113334119319916, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001878639915958047, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1507212370634079, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02388407289981842, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9274546504020691, "metrics/particle/momentum/class_1/accepted/samples": 15969.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08688778430223465, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24964962899684906, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3253011405467987, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.26456955075263977, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04977533966302872, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.333368301391602, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006127093452960253, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3357897996902466, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06560678780078888, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5732274055480957, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9315172433853149, "metrics/particle/momentum/class_1/event/energy_bias": 1.3935106992721558, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.623817443847656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.4234795570373535, "metrics/particle/momentum/class_1/event/px_bias": 0.389289915561676, "metrics/particle/momentum/class_1/event/py_bias": 0.21653908491134644, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.423625946044922, "metrics/particle/momentum/class_1/event/pz_bias": -2.1393814086914062, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.312191009521484, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03925815969705582, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19853511452674866, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20120815932750702, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05479481816291809, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24152474105358124, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.05090595409274101, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.923396348953247, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.30898207426071167, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.494266033172607, "metrics/particle/momentum/class_2/accepted/samples": 1159.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9289984703063965, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9130449891090393, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 6833.7548828125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 6833.251953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3929421007633209, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 343125.03125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0908009484410286, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4089996814727783, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1310439258813858, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.067071914672852, "metrics/particle/momentum/class_2/assigned/samples": 2522.0, "metrics/particle/momentum/class_2/efficiency": 0.4010380506515503, "metrics/particle/momentum/class_2/event/energy_bias": -9.483048439025879, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.602020263671875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.225324630737305, "metrics/particle/momentum/class_2/event/px_bias": -0.3340166211128235, "metrics/particle/momentum/class_2/event/py_bias": -1.0866246223449707, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.112334251403809, "metrics/particle/momentum/class_2/event/pz_bias": -1.1132627725601196, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.988039016723633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02849634550511837, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13440509140491486, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1336435228586197, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.012664261274039745, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1754252314567566, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007656316738575697, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17730283737182617, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00013040861813351512, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7414638996124268, "metrics/particle/momentum/class_3/accepted/samples": 11755.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3104904890060425, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5555058717727661, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.470886468887329, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3253583908081055, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14506596326828003, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.523377418518066, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008108850568532944, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5102085471153259, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.011310901492834091, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7733311653137207, "metrics/particle/momentum/class_3/assigned/samples": 14552.0, "metrics/particle/momentum/class_3/efficiency": 0.7739153504371643, "metrics/particle/momentum/class_3/event/energy_bias": 5.603018283843994, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.917678833007812, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.767692565917969, "metrics/particle/momentum/class_3/event/px_bias": -0.24105434119701385, "metrics/particle/momentum/class_3/event/py_bias": -1.0856482982635498, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.835507869720459, "metrics/particle/momentum/class_3/event/pz_bias": 0.990104615688324, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.860169410705566, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012998883612453938, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16610467433929443, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10550753027200699, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04757068678736687, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15554100275039673, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006644518114626408, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4282713830471039, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.03698688745498657, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3574987649917603, "metrics/particle/momentum/class_4/accepted/samples": 301.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.18501314520835876, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.750558614730835, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8742886781692505, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8100682497024536, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.13031160831451416, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.08356761932373, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01133144460618496, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5225911140441895, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05443600192666054, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6304841041564941, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8502824902534485, "metrics/particle/momentum/class_4/event/energy_bias": -1.6069295406341553, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3961234092712402, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.714113712310791, "metrics/particle/momentum/class_4/event/px_bias": -0.059520479291677475, "metrics/particle/momentum/class_4/event/py_bias": 0.05087225139141083, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3695414066314697, "metrics/particle/momentum/class_4/event/pz_bias": 0.27607136964797974, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.756149768829346, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006901375949382782, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02135932818055153, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021227415651082993, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0005308566032908857, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05881480872631073, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1172550618648529, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.025924013927578926, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.3907899260520935, "metrics/particle/momentum/class_5/accepted/samples": 164.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.018569957464933395, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.048552509397268295, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.047477077692747116, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0017784021329134703, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14463189244270325, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.1870567500591278, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0397668331861496, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.7684755325317383, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9479768872261047, "metrics/particle/momentum/class_5/event/energy_bias": -0.11412646621465683, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9088363647460938, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7423611879348755, "metrics/particle/momentum/class_5/event/px_bias": 0.17725935578346252, "metrics/particle/momentum/class_5/event/py_bias": -0.06326203048229218, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.40842404961586, "metrics/particle/momentum/class_5/event/pz_bias": 0.024322569370269775, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6912195682525635, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5350422859191895, "step_valid_time": 4895.38934469223, "step_total_time": 4896.924387216568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9978321703834478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.089084052741493, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2890, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530685920577617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9612711022840119, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571522742254449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038728897715988087, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9484739087627174, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9566368752068851, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.952537903757416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530685920577617, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9943356105151622, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09324082003723011, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2137, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9411764705882353, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9457432730480811, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9434543454345434, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054256726951918834, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9381035996488147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9426554918394354, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9403740374037404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9411764705882353, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9954941402379838, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0956256105722253, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1307, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1216, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9226100151745068, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9303749043611323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9264761904761905, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06962509563886764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.917298937784522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9250191277735271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9211428571428572, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9226100151745068} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_12000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..271e8e9389296d0c93cfa91322b6b310f13ddd7d --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5683971201212581, "train": {"Classification_binary": 2.839097023010254, "Classification": 0.01990756206214428, "Regression_pt": 0.15704259276390076, "Regression_eta": 0.018609140068292618, "Regression_sin_phi": 0.008542049676179886, "Regression_cos_phi": 0.007845046930015087, "Regression_energy": 0.16122256219387054, "Total": 9.419334411621094, "Total_unweighted": 3.212265976704657}, "train_group": {}, "valid": {"Classification_binary": 2.812541961669922, "Classification": 0.0202518031001091, "Regression_pt": 0.15168194969495138, "Regression_eta": 0.012707379957040152, "Regression_sin_phi": 0.007123620559771855, "Regression_cos_phi": 0.007474775115648906, "Regression_energy": 0.15746764341990152, "Total": 8.897501627604166, "Total_unweighted": 3.169249133517345, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020197520032525063, "diagnostic/loss/cld_pf/Classification_binary": 2.8066823482513428, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0072242687456309795, "diagnostic/loss/cld_pf/Regression_energy": 0.1560848206281662, "diagnostic/loss/cld_pf/Regression_eta": 0.011610627174377441, "diagnostic/loss/cld_pf/Regression_pt": 0.15188388526439667, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006582970265299082, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07406313717365265, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1992049366235733, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04694385826587677, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4401549994945526, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.035621389746665955, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19675923883914948, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06278853118419647, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4366368353366852, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9343209862709045, "metrics/particle/class_1/pid_accuracy": 0.9842259883880615, "metrics/particle/class_2/efficiency": 0.4160875082015991, "metrics/particle/class_2/pid_accuracy": 0.711391031742096, "metrics/particle/class_3/efficiency": 0.7711189985275269, "metrics/particle/class_3/pid_accuracy": 0.9811052083969116, "metrics/particle/class_4/efficiency": 0.8355196714401245, "metrics/particle/class_4/pid_accuracy": 0.25724637508392334, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.5588822364807129, "metrics/particle/count/bias_mean": 0.2947712540626526, "metrics/particle/count/mae": 3.3483660221099854, "metrics/particle/count/prediction_mean": 70.8601303100586, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05857487767934799, "metrics/particle/event/energy_response_mean": 0.9945970773696899, "metrics/particle/event/met_abs_error": 6.167418479919434, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048398762941360474, "metrics/particle/event/scalar_pt_response_mean": 1.0119119882583618, "metrics/particle/event/vector_pt_closure": 0.051822274923324585, "metrics/particle/matched/delta_eta_abs_mean": 0.012386229820549488, "metrics/particle/matched/delta_phi_abs_mean": 0.010781614109873772, "metrics/particle/matched/delta_r_mean": 0.018429087474942207, "metrics/particle/matched/energy_relative_abs_mean": 0.08766570687294006, "metrics/particle/matched/pid_accuracy": 0.9625989198684692, "metrics/particle/matched/pt_relative_abs_mean": 0.08439841866493225, "metrics/particle/matching/duplicate_fraction": 0.03724542632699013, "metrics/particle/matching/efficiency": 0.8226832747459412, "metrics/particle/matching/f1": 0.8209685683250427, "metrics/particle/matching/purity": 0.8192610144615173, "metrics/particle/matching/target_coverage_dr0p05": 0.7822164297103882, "metrics/particle/matching/target_coverage_dr0p10": 0.8570833206176758, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7198258638381958, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8618811368942261, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022703416645526886, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0833483636379242, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08195669203996658, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014712165109813213, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13483093678951263, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005901155527681112, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3871380090713501, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.045964889228343964, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6344584226608276, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8134.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28935903310775757, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.195957064628601, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2565593719482422, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1428251266479492, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12648101150989532, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.250468254089355, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017924049869179726, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8378255367279053, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02599428966641426, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.931349515914917, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9875.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8024070262908936, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.3509163856506348, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.76010513305664, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8443198204040527, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3933054208755493, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.37030643224716187, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.572248458862305, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0369837284088135, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.576827049255371, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.032014813274145126, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09063973277807236, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09191351383924484, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.026424692943692207, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14537134766578674, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.014794888906180859, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8267824053764343, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1628357470035553, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.566222906112671, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1487.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8444266319274902, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8853178024291992, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3718.994140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3718.830322265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.29466718435287476, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 188292.71875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08485791832208633, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.209994077682495, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.35137996077537537, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.476485252380371, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2569.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5409239530563354, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.3709867000579834, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.292804718017578, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1921894550323486, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06553475558757782, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04277323931455612, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.136916160583496, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.191414475440979, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.6274356842041, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015576561912894249, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09070847928524017, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08645867556333542, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007516684476286173, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14190024137496948, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005051780608482659, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11607123911380768, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0016017247689887881, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5130660533905029, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19795.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15804912149906158, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.684637725353241, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5748463273048401, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4494982361793518, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09608225524425507, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.95455265045166, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00363410753197968, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2840268909931183, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002723314333707094, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6215169429779053, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22564.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8658093810081482, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.6847909092903137, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.7063798904418945, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.427498817443848, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.021124720573425293, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.024818703532218933, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.49957537651062, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.17314615845680237, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.3385443687438965, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01837821677327156, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08866981416940689, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08548955619335175, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010462164878845215, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14016111195087433, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027196083683520555, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22695255279541016, "metrics/particle/momentum/all/accepted/pz_bias": -0.02201935462653637, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1174715757369995, "metrics/particle/momentum/all/accepted/samples": 29416.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24545739591121674, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9169796109199524, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 273.63677978515625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 273.5118408203125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11922989040613174, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 51007.25390625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01362545695155859, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6549583673477173, "metrics/particle/momentum/all/assigned/pz_bias": -0.031362541019916534, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.756150245666504, "metrics/particle/momentum/all/assigned/samples": 35008.0, "metrics/particle/momentum/all/efficiency": 0.8228482007980347, "metrics/particle/momentum/all/event/energy_bias": -0.3352794647216797, "metrics/particle/momentum/all/event/p3_error_mean": 16.964879989624023, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.556690216064453, "metrics/particle/momentum/all/event/px_bias": 0.3066459596157074, "metrics/particle/momentum/all/event/py_bias": -0.3523519039154053, "metrics/particle/momentum/all/event/pz_abs_mean": 12.558366775512695, "metrics/particle/momentum/all/event/pz_bias": -2.0552520751953125, "metrics/particle/momentum/all/event/pz_residual_rms": 22.753129959106445, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010358922183513641, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04589494690299034, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04022448882460594, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006917167920619249, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09026264399290085, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016901408089324832, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14794421195983887, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0030749517027288675, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8536357879638672, "metrics/particle/momentum/class_1/accepted/samples": 15975.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08645687997341156, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.28123584389686584, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.34542882442474365, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2876858413219452, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0505191944539547, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.106181621551514, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006125306244939566, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3289570212364197, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01417379081249237, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3135199546813965, "metrics/particle/momentum/class_1/assigned/samples": 17142.0, "metrics/particle/momentum/class_1/efficiency": 0.931867241859436, "metrics/particle/momentum/class_1/event/energy_bias": 3.793663501739502, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.328956604003906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.91416597366333, "metrics/particle/momentum/class_1/event/px_bias": -0.22290033102035522, "metrics/particle/momentum/class_1/event/py_bias": 0.1308630406856537, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.925862312316895, "metrics/particle/momentum/class_1/event/pz_bias": -0.9193152189254761, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.628376007080078, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038689274340867996, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2029825896024704, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20332613587379456, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0575813390314579, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2418411374092102, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.037251655012369156, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7765897512435913, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.30552929639816284, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.07605504989624, "metrics/particle/momentum/class_2/accepted/samples": 1208.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.936690092086792, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9406934380531311, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3632.73828125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3632.2470703125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3900304436683655, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 186167.09375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08447488397359848, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.600260019302368, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.05169015750288963, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.857892036437988, "metrics/particle/momentum/class_2/assigned/samples": 2628.0, "metrics/particle/momentum/class_2/efficiency": 0.41799306869506836, "metrics/particle/momentum/class_2/event/energy_bias": -7.735994338989258, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.469894409179688, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.031207084655762, "metrics/particle/momentum/class_2/event/px_bias": 0.5125571489334106, "metrics/particle/momentum/class_2/event/py_bias": -0.394079327583313, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.153183937072754, "metrics/particle/momentum/class_2/event/pz_bias": -1.2391412258148193, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.824441909790039, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027427155524492264, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.133897602558136, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13508696854114532, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021252494305372238, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17706626653671265, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000509467616211623, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1674354374408722, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01884538307785988, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.579704225063324, "metrics/particle/momentum/class_3/accepted/samples": 11777.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31018638610839844, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6084915399551392, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7414262294769287, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6011297702789307, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15083950757980347, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.043888092041016, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009788593277335167, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5066490769386292, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.043916478753089905, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4303131103515625, "metrics/particle/momentum/class_3/assigned/samples": 14711.0, "metrics/particle/momentum/class_3/efficiency": 0.7753637433052063, "metrics/particle/momentum/class_3/event/energy_bias": 5.1361846923828125, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.641629219055176, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.31120491027832, "metrics/particle/momentum/class_3/event/px_bias": 0.004112184047698975, "metrics/particle/momentum/class_3/event/py_bias": -0.213234081864357, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.832670211791992, "metrics/particle/momentum/class_3/event/pz_bias": -0.24912425875663757, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.99636459350586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013829409144818783, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16883248090744019, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10862140357494354, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05743274837732315, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1555311232805252, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006896551698446274, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5890215039253235, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.05107465758919716, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8906550407409668, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.23496979475021362, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.2146708965301514, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.086620330810547, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.032437562942505, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1666666716337204, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.447824478149414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9536958932876587, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.21398530900478363, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.8845534324645996, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8192090392112732, "metrics/particle/momentum/class_4/event/energy_bias": -1.3485227823257446, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.728788375854492, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9488961696624756, "metrics/particle/momentum/class_4/event/px_bias": -0.15596681833267212, "metrics/particle/momentum/class_4/event/py_bias": 0.0625326931476593, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5880126953125, "metrics/particle/momentum/class_4/event/pz_bias": 0.2352326214313507, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.403507232666016, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008271681144833565, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02447739616036415, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024922126904129982, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0008107488392852247, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07037116587162018, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14339424669742584, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04357907921075821, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.40907952189445496, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.017203498631715775, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04637961834669113, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04572170600295067, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0016181934624910355, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13733838498592377, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21616142988204956, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.015463110990822315, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.7960534691810608, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.18060998618602753, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8224204778671265, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6718258857727051, "metrics/particle/momentum/class_5/event/px_bias": 0.16884373128414154, "metrics/particle/momentum/class_5/event/py_bias": 0.061565790325403214, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.37277060747146606, "metrics/particle/momentum/class_5/event/pz_bias": 0.11709597706794739, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5248284339904785, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5861327648162842, "step_valid_time": 5340.410015106201, "step_total_time": 5341.996148347855, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016374830575379, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09104427272503135, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9501148670823761, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9510512483574245, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9505828271219833, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04894875164257556, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9445356087955366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9454664914586071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9450008208832704, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9501148670823761, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048958471767384, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09437126486958658, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2137, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9381035996488147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9393406593406594, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.938721721941577, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06065934065934066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9337137840210711, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.934945054945055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9343290138370305, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9381035996488147, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086621001688667, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0916857334550476, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1204, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9165402124430956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9193302891933028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9179331306990881, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0806697108066971, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.91350531107739, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9162861491628614, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9148936170212766, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9165402124430956} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_13000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..efc1edd93b063a8cf272fcbae93fbbe188b520d2 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6157635467980296, "train": {"Classification_binary": 2.843076229095459, "Classification": 0.02054588310420513, "Regression_pt": 0.14739388227462769, "Regression_eta": 0.011741525493562222, "Regression_sin_phi": 0.010110379196703434, "Regression_cos_phi": 0.006623866502195597, "Regression_energy": 0.15112890303134918, "Total": 8.902114868164062, "Total_unweighted": 3.1906206686981022}, "train_group": {}, "valid": {"Classification_binary": 2.804997126261393, "Classification": 0.020307087649901707, "Regression_pt": 0.14863873521486917, "Regression_eta": 0.012309063225984573, "Regression_sin_phi": 0.007081795483827591, "Regression_cos_phi": 0.0075858595470587415, "Regression_energy": 0.1537816822528839, "Total": 8.801235834757486, "Total_unweighted": 3.1547013496359186, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020367756485939026, "diagnostic/loss/cld_pf/Classification_binary": 2.7960546016693115, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007222225423902273, "diagnostic/loss/cld_pf/Regression_energy": 0.15303029119968414, "diagnostic/loss/cld_pf/Regression_eta": 0.011240968480706215, "diagnostic/loss/cld_pf/Regression_pt": 0.14896772801876068, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0066672200337052345, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08556505292654037, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20047229528427124, "diagnostic/regression/cld_pf/energy_residual_mean": 0.058445774018764496, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4398483633995056, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.037177328020334244, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1974325329065323, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06434446573257446, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4348423182964325, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9320977926254272, "metrics/particle/class_1/pid_accuracy": 0.984764039516449, "metrics/particle/class_2/efficiency": 0.4191637337207794, "metrics/particle/class_2/pid_accuracy": 0.7105191349983215, "metrics/particle/class_3/efficiency": 0.7640427350997925, "metrics/particle/class_3/pid_accuracy": 0.9832887649536133, "metrics/particle/class_4/efficiency": 0.8244197964668274, "metrics/particle/class_4/pid_accuracy": 0.297429621219635, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.6394422054290771, "metrics/particle/count/bias_mean": -0.6372548937797546, "metrics/particle/count/mae": 3.4895424842834473, "metrics/particle/count/prediction_mean": 69.92810821533203, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05539441108703613, "metrics/particle/event/energy_response_mean": 0.9890720844268799, "metrics/particle/event/met_abs_error": 5.956038951873779, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045479267835617065, "metrics/particle/event/scalar_pt_response_mean": 0.9961031675338745, "metrics/particle/event/vector_pt_closure": 0.05082640424370766, "metrics/particle/matched/delta_eta_abs_mean": 0.010671406053006649, "metrics/particle/matched/delta_phi_abs_mean": 0.012104046531021595, "metrics/particle/matched/delta_r_mean": 0.017983395606279373, "metrics/particle/matched/energy_relative_abs_mean": 0.08827567845582962, "metrics/particle/matched/pid_accuracy": 0.9644570350646973, "metrics/particle/matched/pt_relative_abs_mean": 0.08453231304883957, "metrics/particle/matching/duplicate_fraction": 0.03276941925287247, "metrics/particle/matching/efficiency": 0.8187838792800903, "metrics/particle/matching/f1": 0.8224977254867554, "metrics/particle/matching/purity": 0.8262454271316528, "metrics/particle/matching/target_coverage_dr0p05": 0.7877274751663208, "metrics/particle/matching/target_coverage_dr0p10": 0.853461742401123, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7147038578987122, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8557124733924866, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019648317247629166, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08535180240869522, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08278150856494904, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009952480904757977, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13583828508853912, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005497190169990063, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3800918459892273, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.002105080522596836, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6086673736572266, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8186.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27118125557899475, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8812950849533081, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8939761519432068, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7750806212425232, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12187913805246353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.521463871002197, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015897279605269432, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7687875032424927, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.012991919182240963, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.918762445449829, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9813.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8075367212295532, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.38779231905937195, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.867326259613037, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.389439582824707, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2743893563747406, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09044201672077179, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.862984657287598, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.21981030702590942, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.940502166748047, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02992713823914528, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0962994173169136, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09738709777593613, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02776218391954899, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15014398097991943, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02222222276031971, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.988028347492218, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12150745093822479, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.564216136932373, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1575.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.766124963760376, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5947225093841553, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 34225.359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 34225.19921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2714453637599945, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1729144.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08147277683019638, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9170401096343994, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09230204671621323, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.63234806060791, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2553.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5729356408119202, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0063371658325195, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.256378173828125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1250873804092407, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.06113932281732559, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09511683881282806, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.104221820831299, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7851906418800354, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.75963306427002, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016339393332600594, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09053675830364227, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08578674495220184, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008960871957242489, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1418469101190567, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004604051646310836, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11776429414749146, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010791804641485214, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5191007852554321, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19548.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1695244163274765, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7037218809127808, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5456621050834656, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4212450683116913, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10131607949733734, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.448354244232178, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033905867021530867, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.28454452753067017, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.007629310246556997, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.537976861000061, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22415.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8550059199333191, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.8125492334365845, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.343325614929199, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.16137170791626, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.1939549595117569, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6982862949371338, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.3263673782348633, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.21171067655086517, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.585272312164307, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017993750050663948, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08939827233552933, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08557076007127762, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010248168371617794, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1406581699848175, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030366098508238792, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23779837787151337, "metrics/particle/momentum/all/accepted/pz_bias": 8.024075214052573e-05, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2590327262878418, "metrics/particle/momentum/all/accepted/samples": 29309.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24199733138084412, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.819223165512085, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2512.81884765625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2512.693115234375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11960553377866745, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 468474.0625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012650584802031517, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6143981218338013, "metrics/particle/momentum/all/assigned/pz_bias": 0.005523872561752796, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0655674934387207, "metrics/particle/momentum/all/assigned/samples": 34781.0, "metrics/particle/momentum/all/efficiency": 0.8198550939559937, "metrics/particle/momentum/all/event/energy_bias": -2.431094169616699, "metrics/particle/momentum/all/event/p3_error_mean": 15.625587463378906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.16882038116455, "metrics/particle/momentum/all/event/px_bias": 0.5294836163520813, "metrics/particle/momentum/all/event/py_bias": -0.5127274990081787, "metrics/particle/momentum/all/event/pz_abs_mean": 11.416204452514648, "metrics/particle/momentum/all/event/pz_bias": -1.2167115211486816, "metrics/particle/momentum/all/event/pz_residual_rms": 19.10993194580078, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01066888403147459, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04683370143175125, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04156515747308731, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005388541612774134, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09310591220855713, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015047965571284294, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15073852241039276, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0022654796484857798, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9178269505500793, "metrics/particle/momentum/class_1/accepted/samples": 15949.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08800872415304184, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.29429030418395996, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3503868579864502, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.28946617245674133, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05187605693936348, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.318806171417236, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006127093452960253, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3370780050754547, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.007374194450676441, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.385936975479126, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9303506016731262, "metrics/particle/momentum/class_1/event/energy_bias": 2.845137596130371, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.933170318603516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.546215057373047, "metrics/particle/momentum/class_1/event/px_bias": 0.5131253600120544, "metrics/particle/momentum/class_1/event/py_bias": 0.021583188325166702, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.836889266967773, "metrics/particle/momentum/class_1/event/pz_bias": 0.5436272025108337, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.954309463500977, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.040091078728437424, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19798173010349274, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1986544281244278, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04934367164969444, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23864859342575073, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.043054427951574326, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8249574899673462, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.18221400678157806, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.236063003540039, "metrics/particle/momentum/class_2/accepted/samples": 1231.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.904727041721344, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7535797357559204, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 34089.46875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 34088.984375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.38353490829467773, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1725767.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07608271390199661, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0704846382141113, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04367190599441528, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.2497663497924805, "metrics/particle/momentum/class_2/assigned/samples": 2563.0, "metrics/particle/momentum/class_2/efficiency": 0.4259515702724457, "metrics/particle/momentum/class_2/event/energy_bias": -9.326339721679688, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.638132095336914, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.457076549530029, "metrics/particle/momentum/class_2/event/px_bias": -0.10511350631713867, "metrics/particle/momentum/class_2/event/py_bias": 0.05560123547911644, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.792951583862305, "metrics/particle/momentum/class_2/event/pz_bias": -2.2419168949127197, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.676896095275879, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0258923526853323, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1350240707397461, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13383592665195465, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.022131122648715973, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17607776820659637, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007709439960308373, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1839337944984436, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01175767183303833, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9194915294647217, "metrics/particle/momentum/class_3/accepted/samples": 11674.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3082856237888336, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4268189668655396, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.40095055103302, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2607898712158203, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15294764935970306, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.89840316772461, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00920709129422903, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5114323496818542, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.011658710427582264, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.516953468322754, "metrics/particle/momentum/class_3/assigned/samples": 14554.0, "metrics/particle/momentum/class_3/efficiency": 0.7685825228691101, "metrics/particle/momentum/class_3/event/energy_bias": 5.561007022857666, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.394672393798828, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.398820400238037, "metrics/particle/momentum/class_3/event/px_bias": -0.08278338611125946, "metrics/particle/momentum/class_3/event/py_bias": -0.61749267578125, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.531207084655762, "metrics/particle/momentum/class_3/event/pz_bias": 0.18505096435546875, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.533463478088379, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014317501336336136, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17053154110908508, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11732535809278488, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06023276597261429, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17333881556987762, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010416666977107525, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5053346753120422, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.15293824672698975, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8902831077575684, "metrics/particle/momentum/class_4/accepted/samples": 288.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2674739360809326, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.100403070449829, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.7857955694198608, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7105536460876465, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16101695597171783, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.205303192138672, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6565114855766296, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2756355106830597, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.457124948501587, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8135592937469482, "metrics/particle/momentum/class_4/event/energy_bias": -1.483378529548645, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3695228099823, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7254722118377686, "metrics/particle/momentum/class_4/event/px_bias": 0.07625138759613037, "metrics/particle/momentum/class_4/event/py_bias": 0.06200948357582092, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3056670427322388, "metrics/particle/momentum/class_4/event/pz_bias": 0.1697111874818802, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.701820373535156, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008849551901221275, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02469266764819622, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025969071313738823, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.003245400032028556, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07410095632076263, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15689894556999207, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.05520850792527199, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.461997926235199, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04862900450825691, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05350423976778984, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.048543255776166916, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.011182272806763649, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15165872871875763, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2741715610027313, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.14987896382808685, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2239325046539307, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.02752057835459709, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.921609103679657, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7724950313568115, "metrics/particle/momentum/class_5/event/px_bias": 0.12800376117229462, "metrics/particle/momentum/class_5/event/py_bias": -0.03442869707942009, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.38482755422592163, "metrics/particle/momentum/class_5/event/pz_bias": 0.1268158257007599, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5793019533157349, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6046116352081299, "step_valid_time": 5784.881950616837, "step_total_time": 5786.486562728882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9951075015740595, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08487305552056812, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566787003610109, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9591971043106285, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9579362471245482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0408028956893715, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.951427633738103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.953932214544258, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526782780151166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566787003610109, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970074857407509, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0921846338752641, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2157, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.946883230904302, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9481318681318681, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9475071381506699, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05186813186813187, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9424934152765584, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9437362637362637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9431144300461234, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.946883230904302, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980796834961674, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09380840033624671, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9218512898330804, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9197577592732779, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.920803334596438, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08024224072672217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9180576631259484, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9159727479182438, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9170140204622963, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9218512898330804} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_14000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..58ec9832134c4f7b3026bf991d5f5b526b3a74cc --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6631299734748011, "train": {"Classification_binary": 2.6523849964141846, "Classification": 0.02002202533185482, "Regression_pt": 0.145888090133667, "Regression_eta": 0.011647402308881283, "Regression_sin_phi": 0.009223352186381817, "Regression_cos_phi": 0.0064013563096523285, "Regression_energy": 0.14981430768966675, "Total": 8.578323364257812, "Total_unweighted": 2.9953815303742886}, "train_group": {}, "valid": {"Classification_binary": 2.757937431335449, "Classification": 0.020098521063725155, "Regression_pt": 0.14735101660092673, "Regression_eta": 0.012415020416180292, "Regression_sin_phi": 0.0069256604959567385, "Regression_cos_phi": 0.007438911745945613, "Regression_energy": 0.15302072962125143, "Total": 8.705719629923502, "Total_unweighted": 3.105187291279435, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.020240426063537598, "diagnostic/loss/cld_pf/Classification_binary": 2.7476260662078857, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007261569146066904, "diagnostic/loss/cld_pf/Regression_energy": 0.1536702960729599, "diagnostic/loss/cld_pf/Regression_eta": 0.011544723063707352, "diagnostic/loss/cld_pf/Regression_pt": 0.14876778423786163, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006523949094116688, "diagnostic/regression/cld_pf/energy_pred_mean": 0.10487906634807587, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20585091412067413, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07775978744029999, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4448716938495636, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05005185678601265, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19938187301158905, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07721899449825287, "diagnostic/regression/cld_pf/pt_residual_rms": 0.43647125363349915, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9337843656539917, "metrics/particle/class_1/pid_accuracy": 0.9860230088233948, "metrics/particle/class_2/efficiency": 0.4125555455684662, "metrics/particle/class_2/pid_accuracy": 0.7230046987533569, "metrics/particle/class_3/efficiency": 0.7569884061813354, "metrics/particle/class_3/pid_accuracy": 0.9804331660270691, "metrics/particle/class_4/efficiency": 0.8183652758598328, "metrics/particle/class_4/pid_accuracy": 0.3045622706413269, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.56175297498703, "metrics/particle/count/bias_mean": -1.388235330581665, "metrics/particle/count/mae": 3.577777862548828, "metrics/particle/count/prediction_mean": 69.1771240234375, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.056298982352018356, "metrics/particle/event/energy_response_mean": 0.9869686365127563, "metrics/particle/event/met_abs_error": 5.927464962005615, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04395230486989021, "metrics/particle/event/scalar_pt_response_mean": 0.9948915839195251, "metrics/particle/event/vector_pt_closure": 0.049762941896915436, "metrics/particle/matched/delta_eta_abs_mean": 0.011936984956264496, "metrics/particle/matched/delta_phi_abs_mean": 0.011051399633288383, "metrics/particle/matched/delta_r_mean": 0.018227452412247658, "metrics/particle/matched/energy_relative_abs_mean": 0.09308261424303055, "metrics/particle/matched/pid_accuracy": 0.964337170124054, "metrics/particle/matched/pt_relative_abs_mean": 0.08584979921579361, "metrics/particle/matching/duplicate_fraction": 0.033068470656871796, "metrics/particle/matching/efficiency": 0.8160329461097717, "metrics/particle/matching/f1": 0.8241396546363831, "metrics/particle/matching/purity": 0.8324089646339417, "metrics/particle/matching/target_coverage_dr0p05": 0.7870606184005737, "metrics/particle/matching/target_coverage_dr0p10": 0.852794885635376, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7101653218269348, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8510721325874329, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02062704972922802, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0905521884560585, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08386415243148804, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019058378413319588, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13677644729614258, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006298629101365805, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.40804359316825867, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03350435942411423, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6609649658203125, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8097.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2729155719280243, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0338897705078125, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0954773426055908, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9865986704826355, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12292803823947906, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.393723487854004, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01626685820519924, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.790479838848114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.033787570893764496, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.869431257247925, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9713.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.798757016658783, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.02523571252822876, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.3177170753479, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.153067111968994, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.022566882893443108, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.11592056602239609, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.38711404800415, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.24678316712379456, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.444046974182129, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029980730265378952, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1013810858130455, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09735129773616791, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016064491122961044, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14993803203105927, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02278481051325798, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9750106334686279, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3027168810367584, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9996120929718018, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1580.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7365605235099792, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6743929386138916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9967.15625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9966.9873046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26378896832466125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 498404.34375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07673861086368561, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8575148582458496, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3243796229362488, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.283564567565918, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2502.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5747544765472412, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.5881831645965576, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.617814064025879, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0642107725143433, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0836702287197113, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03768501058220863, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.456896781921387, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.755681037902832, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.204294204711914, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016307607293128967, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09500966966152191, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08757533133029938, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.020564353093504906, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14247557520866394, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006668376736342907, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12346334755420685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005113434512168169, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5349679589271545, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19495.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.157983660697937, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6178559064865112, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4651375114917755, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3563120663166046, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10060921311378479, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.9994888305664062, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0036283819936215878, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.287043035030365, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0018174827564507723, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5713456869125366, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22324.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8526877760887146, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.06165879964828491, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.431978702545166, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.2190399169921875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.12956969439983368, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.19367361068725586, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.4045612812042236, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.011916179209947586, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.683706283569336, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018247071653604507, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09411752969026566, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08707473427057266, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019902635365724564, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14133556187152863, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0034279446117579937, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24857273697853088, "metrics/particle/momentum/all/accepted/pz_bias": -0.02911229617893696, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2017979621887207, "metrics/particle/momentum/all/accepted/samples": 29172.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23221664130687714, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8113877177238464, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 722.6280517578125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 722.5147705078125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11870639026165009, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 134143.796875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012478647753596306, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6148233413696289, "metrics/particle/momentum/all/assigned/pz_bias": -0.0318249836564064, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9802660942077637, "metrics/particle/momentum/all/assigned/samples": 34539.0, "metrics/particle/momentum/all/efficiency": 0.8160228133201599, "metrics/particle/momentum/all/event/energy_bias": -2.6246063709259033, "metrics/particle/momentum/all/event/p3_error_mean": 15.948392868041992, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.161603927612305, "metrics/particle/momentum/all/event/px_bias": 0.19067302346229553, "metrics/particle/momentum/all/event/py_bias": 0.04006805270910263, "metrics/particle/momentum/all/event/pz_abs_mean": 11.825069427490234, "metrics/particle/momentum/all/event/pz_bias": -2.01438045501709, "metrics/particle/momentum/all/event/pz_residual_rms": 19.913978576660156, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010587424039840698, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05310456082224846, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04274445027112961, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012472170405089855, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09186296164989471, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002001250861212611, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1622616946697235, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.026552880182862282, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.89997398853302, "metrics/particle/momentum/class_1/accepted/samples": 15990.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08628379553556442, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2584775388240814, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.29801806807518005, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24384482204914093, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.049244415014982224, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.379375457763672, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006068031769245863, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3450329899787903, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.040012482553720474, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.39436936378479, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9327422380447388, "metrics/particle/momentum/class_1/event/energy_bias": 3.9695165157318115, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.648568153381348, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.458307266235352, "metrics/particle/momentum/class_1/event/px_bias": -0.15472984313964844, "metrics/particle/momentum/class_1/event/py_bias": 0.35051068663597107, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.458270072937012, "metrics/particle/momentum/class_1/event/pz_bias": -1.0504567623138428, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.384950637817383, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03974330797791481, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20001356303691864, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20124290883541107, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.057704146951436996, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24012014269828796, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.048281505703926086, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8917759656906128, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.30664464831352234, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.402353286743164, "metrics/particle/momentum/class_2/accepted/samples": 1222.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8670898079872131, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.817208468914032, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10147.671875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10147.193359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3707773685455322, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 502947.78125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08180708438158035, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1184327602386475, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2498117983341217, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.8621039390563965, "metrics/particle/momentum/class_2/assigned/samples": 2457.0, "metrics/particle/momentum/class_2/efficiency": 0.42283737659454346, "metrics/particle/momentum/class_2/event/energy_bias": -10.285902976989746, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.345388412475586, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.324145793914795, "metrics/particle/momentum/class_2/event/px_bias": 0.5826215744018555, "metrics/particle/momentum/class_2/event/py_bias": 0.3505995273590088, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.522640228271484, "metrics/particle/momentum/class_2/event/pz_bias": -1.6057524681091309, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.255691528320312, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026876380667090416, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13881461322307587, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13675113022327423, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03776096552610397, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17900711297988892, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000521240581292659, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18667292594909668, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00587556092068553, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6182993054389954, "metrics/particle/momentum/class_3/accepted/samples": 11511.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.29978111386299133, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4499428272247314, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3977411985397339, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2742704153060913, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15822696685791016, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.73875904083252, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008324084803462029, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5105929970741272, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.011555803939700127, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.509510040283203, "metrics/particle/momentum/class_3/assigned/samples": 14416.0, "metrics/particle/momentum/class_3/efficiency": 0.7578510642051697, "metrics/particle/momentum/class_3/event/energy_bias": 4.8568291664123535, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.12775993347168, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.255121231079102, "metrics/particle/momentum/class_3/event/px_bias": -0.07121860980987549, "metrics/particle/momentum/class_3/event/py_bias": -0.6901581883430481, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.308261394500732, "metrics/particle/momentum/class_3/event/pz_bias": 0.6425584554672241, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.623366355895996, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012559706345200539, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17371772229671478, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11323880404233932, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05362873524427414, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16508787870407104, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010638297535479069, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5843296051025391, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06680946797132492, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9354602098464966, "metrics/particle/momentum/class_4/accepted/samples": 282.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24296371638774872, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9088919162750244, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.132459878921509, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.0666847229003906, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16949152946472168, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.804461479187012, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7060384750366211, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.061795737594366074, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2410659790039062, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7966101765632629, "metrics/particle/momentum/class_4/event/energy_bias": -0.9917197823524475, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3110599517822266, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6942527294158936, "metrics/particle/momentum/class_4/event/px_bias": -0.26654455065727234, "metrics/particle/momentum/class_4/event/py_bias": 0.04730362445116043, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2628309726715088, "metrics/particle/momentum/class_4/event/pz_bias": -0.02021487057209015, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.474853515625, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009152616374194622, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03086739405989647, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027940258383750916, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.011343328282237053, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06918222457170486, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18848086893558502, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.007007644046097994, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5033395290374756, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.020921610295772552, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.048930712044239044, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04572147876024246, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010728074237704277, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13551272451877594, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.284537136554718, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06875325739383698, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2161928415298462, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.17332994937896729, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8843209147453308, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7073748111724854, "metrics/particle/momentum/class_5/event/px_bias": 0.10054439306259155, "metrics/particle/momentum/class_5/event/py_bias": -0.018187563866376877, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.41953516006469727, "metrics/particle/momentum/class_5/event/pz_bias": 0.01948528364300728, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.7434033155441284, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6563198566436768, "step_valid_time": 6228.962320566177, "step_total_time": 6230.618640422821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0044728633789552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08705641111174134, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2901, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566787003610109, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9585662610983229, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576215505913273, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04143373890167708, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520840170659666, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539625123314699, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9530223390275953, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566787003610109, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001555432532356, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09166450217180544, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2156, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9550066166740185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9526952695269527, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044993383325981476, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9464442493415277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9510366122629025, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9487348734873488, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9503950834064969, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060950747914534, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09332610529481167, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1312, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1218, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9241274658573596, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9283536585365854, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9262357414448669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07164634146341463, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9226100151745068, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.926829268292683, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9247148288973384, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9241274658573596} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_15000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..2d208a7c6babb05f38d7c93f9bbdba809b83de3a --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7104964001515726, "train": {"Classification_binary": 2.6785154342651367, "Classification": 0.021598219871520996, "Regression_pt": 0.14728614687919617, "Regression_eta": 0.017834534868597984, "Regression_sin_phi": 0.008112956769764423, "Regression_cos_phi": 0.007578039076179266, "Regression_energy": 0.14912214875221252, "Total": 9.003621101379395, "Total_unweighted": 3.030047480482608}, "train_group": {}, "valid": {"Classification_binary": 2.7671496073404946, "Classification": 0.019834468762079876, "Regression_pt": 0.1496556301911672, "Regression_eta": 0.012923469146092733, "Regression_sin_phi": 0.00710541196167469, "Regression_cos_phi": 0.007465372482935588, "Regression_energy": 0.15497567256291708, "Total": 8.7899538675944, "Total_unweighted": 3.119109632447362, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019730735570192337, "diagnostic/loss/cld_pf/Classification_binary": 2.762272834777832, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0070877945981919765, "diagnostic/loss/cld_pf/Regression_energy": 0.1535276174545288, "diagnostic/loss/cld_pf/Regression_eta": 0.011588835157454014, "diagnostic/loss/cld_pf/Regression_pt": 0.14982439577579498, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006550814025104046, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09353374689817429, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20199567079544067, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06641446053981781, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4417998194694519, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04170212149620056, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19664983451366425, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06886925548315048, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4358137547969818, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.93349689245224, "metrics/particle/class_1/pid_accuracy": 0.9846431612968445, "metrics/particle/class_2/efficiency": 0.4175686538219452, "metrics/particle/class_2/pid_accuracy": 0.7225102186203003, "metrics/particle/class_3/efficiency": 0.7846123576164246, "metrics/particle/class_3/pid_accuracy": 0.9819062948226929, "metrics/particle/class_4/efficiency": 0.8415741920471191, "metrics/particle/class_4/pid_accuracy": 0.2925659418106079, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.5826612710952759, "metrics/particle/count/bias_mean": 0.6418300867080688, "metrics/particle/count/mae": 3.445751667022705, "metrics/particle/count/prediction_mean": 71.20719146728516, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.056406158953905106, "metrics/particle/event/energy_response_mean": 0.9983590841293335, "metrics/particle/event/met_abs_error": 6.314296245574951, "metrics/particle/event/scalar_pt_relative_abs_error": 0.047465868294239044, "metrics/particle/event/scalar_pt_response_mean": 1.008879542350769, "metrics/particle/event/vector_pt_closure": 0.05275346338748932, "metrics/particle/matched/delta_eta_abs_mean": 0.012592487968504429, "metrics/particle/matched/delta_phi_abs_mean": 0.011187258176505566, "metrics/particle/matched/delta_r_mean": 0.018874363973736763, "metrics/particle/matched/energy_relative_abs_mean": 0.09196026623249054, "metrics/particle/matched/pid_accuracy": 0.9641187191009521, "metrics/particle/matched/pt_relative_abs_mean": 0.08579855412244797, "metrics/particle/matching/duplicate_fraction": 0.035081278532743454, "metrics/particle/matching/efficiency": 0.8281016945838928, "metrics/particle/matching/f1": 0.8243527412414551, "metrics/particle/matching/purity": 0.820637583732605, "metrics/particle/matching/target_coverage_dr0p05": 0.7992497682571411, "metrics/particle/matching/target_coverage_dr0p10": 0.8645116686820984, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7211225628852844, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8650766611099243, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02050541155040264, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08914594352245331, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0845046415925026, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015411585569381714, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13717545568943024, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005836575757712126, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38984614610671997, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.007342795841395855, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.637241005897522, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8224.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2687286138534546, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.985260009765625, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1005476713180542, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9833890795707703, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12440722435712814, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.581456184387207, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016345474869012833, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.792212188243866, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06282268464565277, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.915811777114868, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9911.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8112853765487671, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6653169393539429, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.751307010650635, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4800326824188232, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2982270419597626, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.30568790435791016, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.634881973266602, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.3334481716156006, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.270732879638672, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029093923047184944, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09347646683454514, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09467267245054245, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.029668856412172318, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1460501253604889, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018471337854862213, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.920468807220459, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08162892609834671, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3231847286224365, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1570.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8113548755645752, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3414175510406494, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5713.970703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5713.8056640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2746013104915619, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 289571.21875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07506806403398514, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9484543800354004, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.020626500248908997, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.306381225585938, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2571.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5711167454719543, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.9195420742034912, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.275872707366943, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.069936990737915, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.061634697020053864, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07207684963941574, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.127106666564941, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6912840604782104, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.16045570373535, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01738121546804905, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09528219699859619, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08788616955280304, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016013406217098236, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14324839413166046, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000554379599634558, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1255393773317337, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.025974571704864502, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5246487259864807, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19842.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.157425656914711, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6023895144462585, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4884391129016876, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.36934345960617065, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0968155711889267, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.820704936981201, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003582485718652606, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2781282961368561, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.038055066019296646, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3012210130691528, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22610.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8678650856018066, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.6182527542114258, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.086932182312012, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.669015407562256, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.659650444984436, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.4386579692363739, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.815523386001587, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.5052911043167114, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.238082408905029, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01886867545545101, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09348372370004654, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08730731159448624, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01656981371343136, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1417408287525177, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029693616088479757, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24099688231945038, "metrics/particle/momentum/all/accepted/pz_bias": 0.011028576642274857, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2301182746887207, "metrics/particle/momentum/all/accepted/samples": 29636.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23677068948745728, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7646678686141968, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 419.257080078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 419.1351623535156, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1176336482167244, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 78379.4765625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012424483895301819, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6189607977867126, "metrics/particle/momentum/all/assigned/pz_bias": 0.04075087979435921, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.136475086212158, "metrics/particle/momentum/all/assigned/samples": 35092.0, "metrics/particle/momentum/all/efficiency": 0.8290022015571594, "metrics/particle/momentum/all/event/energy_bias": 0.36402755975723267, "metrics/particle/momentum/all/event/p3_error_mean": 16.215211868286133, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.86031436920166, "metrics/particle/momentum/all/event/px_bias": 0.2997887432575226, "metrics/particle/momentum/all/event/py_bias": 0.6722689867019653, "metrics/particle/momentum/all/event/pz_abs_mean": 11.616608619689941, "metrics/particle/momentum/all/event/pz_bias": 3.5300235748291016, "metrics/particle/momentum/all/event/pz_residual_rms": 19.562572479248047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01177586056292057, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.051344942301511765, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04180959239602089, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009595032781362534, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0920427069067955, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019399248994886875, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16804184019565582, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.020757559686899185, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.046273112297058, "metrics/particle/momentum/class_1/accepted/samples": 15980.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08592656254768372, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.26208943128585815, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30924028158187866, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25409233570098877, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05028878152370453, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.753541946411133, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006067323964089155, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3445075750350952, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.035869453102350235, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.288614511489868, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.932158887386322, "metrics/particle/momentum/class_1/event/energy_bias": 3.7998249530792236, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.420564651489258, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.286603927612305, "metrics/particle/momentum/class_1/event/px_bias": 0.6769058704376221, "metrics/particle/momentum/class_1/event/py_bias": -0.18817473948001862, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.319221496582031, "metrics/particle/momentum/class_1/event/pz_bias": 1.6232335567474365, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.9833345413208, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03857330605387688, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19708196818828583, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1979503482580185, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05156246945261955, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23710858821868896, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.040749795734882355, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7941621541976929, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14444975554943085, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.308549404144287, "metrics/particle/momentum/class_2/accepted/samples": 1227.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9522700905799866, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.794909656047821, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5577.75439453125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5577.24755859375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3801746964454651, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 286141.59375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0805165246129036, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.2381672859191895, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.007578785065561533, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.022562026977539, "metrics/particle/momentum/class_2/assigned/samples": 2633.0, "metrics/particle/momentum/class_2/efficiency": 0.42456746101379395, "metrics/particle/momentum/class_2/event/energy_bias": -9.13368034362793, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.965842247009277, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.69419002532959, "metrics/particle/momentum/class_2/event/px_bias": -0.4068654179573059, "metrics/particle/momentum/class_2/event/py_bias": 1.1123942136764526, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.922672271728516, "metrics/particle/momentum/class_2/event/pz_bias": 0.1902962327003479, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.11181640625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026561647653579712, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1377176195383072, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1368962675333023, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03160903602838516, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17870765924453735, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00041785056237131357, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17354314029216766, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.010994828306138515, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5653536319732666, "metrics/particle/momentum/class_3/accepted/samples": 11966.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.28752490878105164, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.330209732055664, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.383670449256897, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2504897117614746, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14946258068084717, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.345749855041504, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007841547019779682, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.47386640310287476, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04924779012799263, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.386023759841919, "metrics/particle/momentum/class_3/assigned/samples": 14793.0, "metrics/particle/momentum/class_3/efficiency": 0.7878069877624512, "metrics/particle/momentum/class_3/event/energy_bias": 7.104291915893555, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.568867683410645, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.596887588500977, "metrics/particle/momentum/class_3/event/px_bias": -0.16892442107200623, "metrics/particle/momentum/class_3/event/py_bias": -0.4929904639720917, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.572214126586914, "metrics/particle/momentum/class_3/event/pz_bias": 1.3801571130752563, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.086915016174316, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014009029604494572, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18829186260700226, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11515498906373978, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07412414997816086, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16530220210552216, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006756756920367479, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4967711269855499, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.059677716344594955, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.540307879447937, "metrics/particle/momentum/class_4/accepted/samples": 296.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2026723474264145, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5965272188186646, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6075743436813354, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5474803447723389, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.13920454680919647, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.700274467468262, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008522727526724339, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6487697958946228, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07139825075864792, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0550193786621094, "metrics/particle/momentum/class_4/assigned/samples": 352.0, "metrics/particle/momentum/class_4/efficiency": 0.8361582159996033, "metrics/particle/momentum/class_4/event/energy_bias": -1.3298487663269043, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.412574291229248, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7671139240264893, "metrics/particle/momentum/class_4/event/px_bias": 0.13509555160999298, "metrics/particle/momentum/class_4/event/py_bias": 0.17201313376426697, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.356394648551941, "metrics/particle/momentum/class_4/event/pz_bias": 0.11463768035173416, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.7200422286987305, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01018572598695755, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02699660137295723, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025452746078372, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004952281713485718, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06914980709552765, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19027456641197205, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1386118233203888, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5326756834983826, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.022347357124090195, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.049024906009435654, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.047583408653736115, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.005752321798354387, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1456785649061203, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2947958707809448, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.24035786092281342, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2441766262054443, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.07655984908342361, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0865029096603394, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7974547743797302, "metrics/particle/momentum/class_5/event/px_bias": 0.06357716023921967, "metrics/particle/momentum/class_5/event/py_bias": 0.06902692466974258, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5748171210289001, "metrics/particle/momentum/class_5/event/pz_bias": 0.2216988205909729, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.970613479614258, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6212139129638672, "step_valid_time": 6671.263170480728, "step_total_time": 6672.884384632111, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0076739682977183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08844386553558004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2897, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2879, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9507712504102396, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9542160737812911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524905474272563, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045783926218708824, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9448638004594684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9482872200263505, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9465724149268453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9507712504102396, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063940698292586, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09440448881678098, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2151, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9503950834064969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.950812472551603, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9506037321624589, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049187527448397016, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9442493415276558, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9446640316205533, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.944456641053787, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9503950834064969, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070671407403315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08968523976215514, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1218, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9299847792998478, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9285714285714286, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0700152207001522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9241274658573596, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9269406392694064, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.925531914893617, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9271623672230652} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_16000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5618856d335828af29cc764b723255f994d78c --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.757862826828344, "train": {"Classification_binary": 2.5479331016540527, "Classification": 0.01941506750881672, "Regression_pt": 0.13605858385562897, "Regression_eta": 0.012059887871146202, "Regression_sin_phi": 0.005768795497715473, "Regression_cos_phi": 0.007320880889892578, "Regression_energy": 0.13955920934677124, "Total": 8.082230567932129, "Total_unweighted": 2.868115526624024}, "train_group": {}, "valid": {"Classification_binary": 2.7085224787394204, "Classification": 0.019595323751370113, "Regression_pt": 0.1464216113090515, "Regression_eta": 0.012078980604807535, "Regression_sin_phi": 0.007069141914447148, "Regression_cos_phi": 0.007401125505566597, "Regression_energy": 0.15209177136421204, "Total": 8.601518630981445, "Total_unweighted": 3.053180433188875, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.019491877406835556, "diagnostic/loss/cld_pf/Classification_binary": 2.698486804962158, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007092997897416353, "diagnostic/loss/cld_pf/Regression_energy": 0.15141618251800537, "diagnostic/loss/cld_pf/Regression_eta": 0.011154592968523502, "diagnostic/loss/cld_pf/Regression_pt": 0.1470348834991455, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006587512791156769, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07638566941022873, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1957242786884308, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04926638305187225, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42580756545066833, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03012465499341488, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19112403690814972, "diagnostic/regression/cld_pf/pt_residual_mean": 0.057291798293590546, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4205350875854492, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9399555325508118, "metrics/particle/class_1/pid_accuracy": 0.9847078919410706, "metrics/particle/class_2/efficiency": 0.42520222067832947, "metrics/particle/class_2/pid_accuracy": 0.7154340744018555, "metrics/particle/class_3/efficiency": 0.7776898741722107, "metrics/particle/class_3/pid_accuracy": 0.984005868434906, "metrics/particle/class_4/efficiency": 0.8203834295272827, "metrics/particle/class_4/pid_accuracy": 0.339483380317688, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6143141388893127, "metrics/particle/count/bias_mean": -0.2934640645980835, "metrics/particle/count/mae": 3.3222222328186035, "metrics/particle/count/prediction_mean": 70.27189636230469, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05604112893342972, "metrics/particle/event/energy_response_mean": 0.9702106714248657, "metrics/particle/event/met_abs_error": 5.281218528747559, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044237397611141205, "metrics/particle/event/scalar_pt_response_mean": 0.9830933213233948, "metrics/particle/event/vector_pt_closure": 0.046906016767024994, "metrics/particle/matched/delta_eta_abs_mean": 0.011502048932015896, "metrics/particle/matched/delta_phi_abs_mean": 0.011068540625274181, "metrics/particle/matched/delta_r_mean": 0.017893848940730095, "metrics/particle/matched/energy_relative_abs_mean": 0.0892312228679657, "metrics/particle/matched/pid_accuracy": 0.9652552008628845, "metrics/particle/matched/pt_relative_abs_mean": 0.08396420627832413, "metrics/particle/matching/duplicate_fraction": 0.03057219460606575, "metrics/particle/matching/efficiency": 0.82879638671875, "metrics/particle/matching/f1": 0.8305233120918274, "metrics/particle/matching/purity": 0.8322575092315674, "metrics/particle/matching/target_coverage_dr0p05": 0.7993979454040527, "metrics/particle/matching/target_coverage_dr0p10": 0.8622238636016846, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7238271832466125, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8651692867279053, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019713623449206352, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08437015861272812, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08168647438287735, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01438030507415533, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1339186429977417, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0053378622978925705, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3816576898097992, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.016678061336278915, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5863010883331299, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8243.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25389817357063293, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8340291976928711, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8183570504188538, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7040928602218628, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11588462442159653, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.166750431060791, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01411740854382515, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7410373091697693, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.012949646450579166, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3485841751098633, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9846.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.813159704208374, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.412900686264038, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.772317886352539, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2599101066589355, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1480170488357544, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1407552808523178, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.828425884246826, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7118320465087891, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.772089958190918, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02872535027563572, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09066510200500488, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09092628210783005, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017062261700630188, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1409643590450287, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016331657767295837, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.7601744532585144, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1315522938966751, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.4037632942199707, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1592.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7355088591575623, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1319481134414673, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10371.0927734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10370.9189453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26868608593940735, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 522772.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07907159626483917, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.797840118408203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2516830861568451, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.782247543334961, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2542.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5791196823120117, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.038590908050537, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.815794944763184, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1519672870635986, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07789111137390137, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10292381793260574, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.650091648101807, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.4325934648513794, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.555124282836914, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016453972086310387, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09235444664955139, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08561376482248306, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.00978564377874136, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14116553962230682, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000605387962423265, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1202879324555397, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02117273211479187, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5448606610298157, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19822.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15139661729335785, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5684053301811218, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4485531151294708, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3280322849750519, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09374583512544632, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.778722763061523, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027114725671708584, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.25831592082977295, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01783006638288498, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6484030485153198, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22497.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.866990327835083, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.5693275928497314, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.952589988708496, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.758139610290527, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.3373095989227295, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.31581196188926697, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2425594329833984, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5331158638000488, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.953669548034668, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018018707633018494, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09004456549882889, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0848073661327362, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011453316546976566, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13917821645736694, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002764945849776268, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22728359699249268, "metrics/particle/momentum/all/accepted/pz_bias": -0.025848684832453728, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0990897417068481, "metrics/particle/momentum/all/accepted/samples": 29657.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22288990020751953, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6844397187232971, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 756.240966796875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 756.118408203125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1127418652176857, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 141117.453125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011494912207126617, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5796100497245789, "metrics/particle/momentum/all/assigned/pz_bias": -0.033493008464574814, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2461130619049072, "metrics/particle/momentum/all/assigned/samples": 34885.0, "metrics/particle/momentum/all/efficiency": 0.8295896649360657, "metrics/particle/momentum/all/event/energy_bias": -8.020818710327148, "metrics/particle/momentum/all/event/p3_error_mean": 14.834390640258789, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.560218334197998, "metrics/particle/momentum/all/event/px_bias": -0.26718372106552124, "metrics/particle/momentum/all/event/py_bias": 0.5594910383224487, "metrics/particle/momentum/all/event/pz_abs_mean": 11.033649444580078, "metrics/particle/momentum/all/event/pz_bias": -2.677541494369507, "metrics/particle/momentum/all/event/pz_residual_rms": 19.66803741455078, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009647813625633717, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04973108693957329, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04039148613810539, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0049961465410888195, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09048351645469666, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015544363996014, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14630508422851562, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.019311584532260895, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8330518007278442, "metrics/particle/momentum/class_1/accepted/samples": 16083.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07846888899803162, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1982189416885376, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.24293272197246552, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18427129089832306, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04346304014325142, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.6662986278533936, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005250568967312574, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31215667724609375, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.023311078548431396, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5253520011901855, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.938167154788971, "metrics/particle/momentum/class_1/event/energy_bias": -0.2914906442165375, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.789478302001953, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.7484660148620605, "metrics/particle/momentum/class_1/event/px_bias": -0.19303080439567566, "metrics/particle/momentum/class_1/event/py_bias": 0.21871966123580933, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.083260536193848, "metrics/particle/momentum/class_1/event/pz_bias": -1.1574050188064575, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.84621238708496, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04108729213476181, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19936630129814148, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20021037757396698, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07569460570812225, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23999077081680298, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03585657477378845, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7099812030792236, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.21060030162334442, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.886784553527832, "metrics/particle/momentum/class_2/accepted/samples": 1255.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8772019743919373, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7495748400688171, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10177.4921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10176.9755859375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37528958916664124, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 517905.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07953667640686035, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0851874351501465, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19728395342826843, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.07705545425415, "metrics/particle/momentum/class_2/assigned/samples": 2590.0, "metrics/particle/momentum/class_2/efficiency": 0.43425604701042175, "metrics/particle/momentum/class_2/event/energy_bias": -12.268473625183105, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.4965181350708, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.196226596832275, "metrics/particle/momentum/class_2/event/px_bias": -0.03415548801422119, "metrics/particle/momentum/class_2/event/py_bias": 0.39752477407455444, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.863316535949707, "metrics/particle/momentum/class_2/event/pz_bias": -2.162640333175659, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.660942077636719, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027218418195843697, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13236698508262634, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13293175399303436, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02868921309709549, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17451894283294678, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006741383695043623, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17232699692249298, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.019803045317530632, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6050559282302856, "metrics/particle/momentum/class_3/accepted/samples": 11867.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27727630734443665, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2064621448516846, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.152141809463501, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.022438645362854, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14712518453598022, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.801061630249023, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006768305320292711, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4504658281803131, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.02336595207452774, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.939418077468872, "metrics/particle/momentum/class_3/assigned/samples": 14627.0, "metrics/particle/momentum/class_3/efficiency": 0.7812891006469727, "metrics/particle/momentum/class_3/event/energy_bias": 5.690909385681152, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.4306058883667, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.2910590171813965, "metrics/particle/momentum/class_3/event/px_bias": -0.12358132004737854, "metrics/particle/momentum/class_3/event/py_bias": -0.18698036670684814, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.63808536529541, "metrics/particle/momentum/class_3/event/pz_bias": 0.18283182382583618, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.820591926574707, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011947287246584892, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.160039022564888, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11649332195520401, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.045346468687057495, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17151856422424316, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.014084506779909134, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6024435758590698, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.16414465010166168, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.164188861846924, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28003790974617004, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4892120361328125, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.069286346435547, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9905476570129395, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16949152946472168, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.758685111999512, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.695601224899292, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.19694770872592926, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.334805488586426, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.2404472827911377, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4149138927459717, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7880561351776123, "metrics/particle/momentum/class_4/event/px_bias": -0.06898816674947739, "metrics/particle/momentum/class_4/event/py_bias": 0.11160063743591309, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3303567171096802, "metrics/particle/momentum/class_4/event/pz_bias": 0.31230002641677856, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.364345550537109, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00747882341966033, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02483225427567959, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02182641811668873, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005841853097081184, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06585285812616348, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15118782222270966, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01974504254758358, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4253269135951996, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02122463844716549, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05495627224445343, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05145822465419769, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.005454185418784618, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.22911258041858673, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2495713233947754, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.0820307657122612, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.1970243453979492, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.08868321031332016, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9645568132400513, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.718544065952301, "metrics/particle/momentum/class_5/event/px_bias": 0.15257211029529572, "metrics/particle/momentum/class_5/event/py_bias": 0.018626250326633453, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4836888313293457, "metrics/particle/momentum/class_5/event/pz_bias": 0.14737196266651154, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.838587760925293, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5293316841125488, "step_valid_time": 7119.767552137375, "step_total_time": 7121.296884059906, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9934184241679961, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08663762782895013, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2891, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530685920577617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954949029924367, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9540078843626807, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045050970075633016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9488021004266491, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506741203551463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497371879106439, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530685920577617, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9941202259492707, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09053154435793154, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2281, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2158, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9473222124670764, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9460762823323103, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466988374643562, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05392371766768961, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9460052677787533, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9447610697062692, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9453827593770564, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9473222124670764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9953102892579615, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.090446899512868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1218, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9241274658573596, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9297709923664123, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9269406392694064, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07022900763358779, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9233687405159332, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9290076335877863, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.926179604261796, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9241274658573596} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_17000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..6d1955fa2b3937038feb46073320cd3d759ac339 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8052292535051155, "train": {"Classification_binary": 2.695769786834717, "Classification": 0.020359788089990616, "Regression_pt": 0.14564833045005798, "Regression_eta": 0.01679827831685543, "Regression_sin_phi": 0.006340770982205868, "Regression_cos_phi": 0.006057078950107098, "Regression_energy": 0.15327998995780945, "Total": 8.71934700012207, "Total_unweighted": 3.0442540235817432}, "train_group": {}, "valid": {"Classification_binary": 2.6722262700398765, "Classification": 0.019346447040637333, "Regression_pt": 0.14285186926523843, "Regression_eta": 0.012273646891117096, "Regression_sin_phi": 0.0068290407458941145, "Regression_cos_phi": 0.007216315716505051, "Regression_energy": 0.14829144875208536, "Total": 8.450616200764975, "Total_unweighted": 3.009035038451354, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.0192515067756176, "diagnostic/loss/cld_pf/Classification_binary": 2.674079418182373, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006954193115234375, "diagnostic/loss/cld_pf/Regression_energy": 0.1483107954263687, "diagnostic/loss/cld_pf/Regression_eta": 0.011233555153012276, "diagnostic/loss/cld_pf/Regression_pt": 0.14412467181682587, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006291654892265797, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09917202591896057, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.20046105980873108, "diagnostic/regression/cld_pf/energy_residual_mean": 0.0720527395606041, "diagnostic/regression/cld_pf/energy_residual_rms": 0.44237908720970154, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04455363377928734, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1946471631526947, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07172077149152756, "diagnostic/regression/cld_pf/pt_residual_rms": 0.43368303775787354, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9402621984481812, "metrics/particle/class_1/pid_accuracy": 0.9852020740509033, "metrics/particle/class_2/efficiency": 0.42805057764053345, "metrics/particle/class_2/pid_accuracy": 0.7455416321754456, "metrics/particle/class_3/efficiency": 0.7705256938934326, "metrics/particle/class_3/pid_accuracy": 0.9825167059898376, "metrics/particle/class_4/efficiency": 0.8506559133529663, "metrics/particle/class_4/pid_accuracy": 0.2965599000453949, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6127744317054749, "metrics/particle/count/bias_mean": -0.06928104907274246, "metrics/particle/count/mae": 3.2915031909942627, "metrics/particle/count/prediction_mean": 70.49607849121094, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05473104119300842, "metrics/particle/event/energy_response_mean": 1.00194251537323, "metrics/particle/event/met_abs_error": 5.838033676147461, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044380366802215576, "metrics/particle/event/scalar_pt_response_mean": 1.0041842460632324, "metrics/particle/event/vector_pt_closure": 0.049177177250385284, "metrics/particle/matched/delta_eta_abs_mean": 0.012118280865252018, "metrics/particle/matched/delta_phi_abs_mean": 0.010535724461078644, "metrics/particle/matched/delta_r_mean": 0.017977530136704445, "metrics/particle/matched/energy_relative_abs_mean": 0.08948688954114914, "metrics/particle/matched/pid_accuracy": 0.9654577374458313, "metrics/particle/matched/pt_relative_abs_mean": 0.0838133692741394, "metrics/particle/matching/duplicate_fraction": 0.03338617831468582, "metrics/particle/matching/efficiency": 0.8264159560203552, "metrics/particle/matching/f1": 0.8268218636512756, "metrics/particle/matching/purity": 0.8272281289100647, "metrics/particle/matching/target_coverage_dr0p05": 0.7960913181304932, "metrics/particle/matching/target_coverage_dr0p10": 0.8613254427909851, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7225952744483948, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8636502623558044, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020239397883415222, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08755719661712646, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0831248089671135, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017178762704133987, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13622117042541504, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005490483250468969, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3817809522151947, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.018100721761584282, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5460199117660522, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8196.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2751110792160034, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0820281505584717, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0968166589736938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.987827479839325, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12431500107049942, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.032257080078125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015425208024680614, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7554501295089722, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.012550677172839642, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.843484401702881, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9854.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8085232377052307, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.2300095558166504, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.721109390258789, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.443093776702881, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.24857109785079956, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.025475023314356804, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.6878981590271, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.0830765962600708, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.370842933654785, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0291682668030262, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09626765549182892, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09379054605960846, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019603397697210312, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14454230666160583, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025332488119602203, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9719671010971069, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17677165567874908, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0403482913970947, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1579.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7583942413330078, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5677207708358765, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3232.117919921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3231.9541015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27180686593055725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 163623.40625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08489096909761429, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.899557113647461, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07726720720529556, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.735637664794922, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2568.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.574390709400177, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.062068432569503784, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.5726399421691895, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2152386903762817, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04958898946642876, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.060964539647102356, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.4138078689575195, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.05195188522338867, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.506139755249023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016227837651968002, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09205853193998337, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08556965738534927, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011707929894328117, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14090882241725922, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000606673420406878, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12165803462266922, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010715663433074951, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5198531150817871, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19780.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1554751694202423, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6522026062011719, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4793004095554352, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3634253144264221, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09496063739061356, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.738373279571533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003869590349495411, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2861882448196411, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.00010058055340778083, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7000501155853271, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22483.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8651533126831055, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9580295085906982, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.8205037117004395, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.437196731567383, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5529012084007263, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7097719311714172, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.695814609527588, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6332927942276001, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.640851020812988, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01803164929151535, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09103512018918991, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08533088117837906, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013646887615323067, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13982294499874115, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003282016608864069, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23922209441661835, "metrics/particle/momentum/all/accepted/pz_bias": -0.02163531631231308, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1565238237380981, "metrics/particle/momentum/all/accepted/samples": 29555.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23360693454742432, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8409020304679871, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 238.40890502929688, "metrics/particle/momentum/all/assigned/pt_relative_bias": 238.2914276123047, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1162584125995636, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 44381.20703125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013092679902911186, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6109337210655212, "metrics/particle/momentum/all/assigned/pz_bias": 0.002206250559538603, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.12386155128479, "metrics/particle/momentum/all/assigned/samples": 34905.0, "metrics/particle/momentum/all/efficiency": 0.8267363905906677, "metrics/particle/momentum/all/event/energy_bias": 0.33404862880706787, "metrics/particle/momentum/all/event/p3_error_mean": 15.966500282287598, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.336410522460938, "metrics/particle/momentum/all/event/px_bias": 0.751883327960968, "metrics/particle/momentum/all/event/py_bias": -0.7452614307403564, "metrics/particle/momentum/all/event/pz_abs_mean": 11.755511283874512, "metrics/particle/momentum/all/event/pz_bias": -0.602168083190918, "metrics/particle/momentum/all/event/pz_residual_rms": 20.836341857910156, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010526790283620358, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.047911323606967926, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04011610522866249, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004084405954927206, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08980395644903183, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013666293816640973, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14766736328601837, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.027764353901147842, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8049220442771912, "metrics/particle/momentum/class_1/accepted/samples": 16098.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07751873880624771, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23603211343288422, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2907436192035675, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23286372423171997, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04456889629364014, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.2912774085998535, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00530859874561429, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3141807019710541, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.005334301386028528, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.530026435852051, "metrics/particle/momentum/class_1/assigned/samples": 17142.0, "metrics/particle/momentum/class_1/efficiency": 0.9390421509742737, "metrics/particle/momentum/class_1/event/energy_bias": 0.901881217956543, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.494132995605469, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.71687126159668, "metrics/particle/momentum/class_1/event/px_bias": 0.14871002733707428, "metrics/particle/momentum/class_1/event/py_bias": -0.5399367809295654, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.78712272644043, "metrics/particle/momentum/class_1/event/pz_bias": -0.6700972318649292, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.494836807250977, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03846251219511032, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19380761682987213, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19495850801467896, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.056486669927835464, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.235102117061615, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04948939383029938, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8679643869400024, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14761528372764587, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.324120998382568, "metrics/particle/momentum/class_2/accepted/samples": 1273.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9022546410560608, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7768920660018921, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3207.420654296875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3206.952392578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3723897933959961, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 163052.953125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08275328576564789, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.044278860092163, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.06873975694179535, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.7351555824279785, "metrics/particle/momentum/class_2/assigned/samples": 2586.0, "metrics/particle/momentum/class_2/efficiency": 0.4404844343662262, "metrics/particle/momentum/class_2/event/energy_bias": -5.642264366149902, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.213565826416016, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.417972564697266, "metrics/particle/momentum/class_2/event/px_bias": 0.22174739837646484, "metrics/particle/momentum/class_2/event/py_bias": 0.26820099353790283, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.241517066955566, "metrics/particle/momentum/class_2/event/pz_bias": -1.210929274559021, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.619441032409668, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02635502815246582, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1379544883966446, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13548342883586884, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03332449123263359, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1772075742483139, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007679180707782507, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18096648156642914, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0002686749503482133, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6071276664733887, "metrics/particle/momentum/class_3/accepted/samples": 11720.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30054885149002075, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5627974271774292, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4949299097061157, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.367434024810791, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15542662143707275, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.97363567352295, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010034129954874516, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5317342877388, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.002744264667853713, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7607526779174805, "metrics/particle/momentum/class_3/assigned/samples": 14650.0, "metrics/particle/momentum/class_3/efficiency": 0.7716110348701477, "metrics/particle/momentum/class_3/event/energy_bias": 6.711462497711182, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.320779800415039, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.3360595703125, "metrics/particle/momentum/class_3/event/px_bias": 0.23887716233730316, "metrics/particle/momentum/class_3/event/py_bias": -0.6473937034606934, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.5346174240112305, "metrics/particle/momentum/class_3/event/pz_bias": 0.8378117680549622, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.82913589477539, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014993622899055481, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17530056834220886, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.122904933989048, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0653233453631401, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17342835664749146, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010067113675177097, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5670469403266907, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.019360529258847237, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.783545732498169, "metrics/particle/momentum/class_4/accepted/samples": 298.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.234116792678833, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.1140137910842896, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.078872561454773, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.0181167125701904, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.326148986816406, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6184710264205933, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.034180834889411926, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8478604555130005, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8418079018592834, "metrics/particle/momentum/class_4/event/energy_bias": -1.4998937845230103, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.159072160720825, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.595925211906433, "metrics/particle/momentum/class_4/event/px_bias": 0.008189579471945763, "metrics/particle/momentum/class_4/event/py_bias": 0.18001455068588257, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1675848960876465, "metrics/particle/momentum/class_4/event/pz_bias": 0.38095447421073914, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.225337028503418, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0069481385871768, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020984072238206863, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02103283256292343, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.003246271051466465, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06131525710225105, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15201109647750854, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.04329860210418701, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4304693639278412, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03511407971382141, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04182608798146248, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04012618213891983, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.005967402830719948, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1326654702425003, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.33296120166778564, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.10862548649311066, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.0399270057678223, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.13713708519935608, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7015127539634705, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5840988159179688, "metrics/particle/momentum/class_5/event/px_bias": 0.13435915112495422, "metrics/particle/momentum/class_5/event/py_bias": -0.006146435625851154, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.30797868967056274, "metrics/particle/momentum/class_5/event/pz_bias": 0.0600920245051384, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.336274266242981, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6321580410003662, "step_valid_time": 7567.83079624176, "step_total_time": 7569.462954759598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014744841212853, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08257999974404628, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2909, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2898, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547095503774204, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528332787422208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537704918032787, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04716672125777923, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9510994420741713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9492302653128071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9501639344262295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547095503774204, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024682657015198, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0903903524783305, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2159, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9512730465320457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9542051959489212, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527368652451088, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04579480405107882, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9477611940298507, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506825187142228, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9492196087052099, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9512730465320457, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058607486372397, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08412165325519827, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9301972685887708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9337395277989338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9319650323071076, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06626047220106626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9306930693069307, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9289243633599392, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9301972685887708} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_18000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..d58af671080eca335a4c562a5a2478641b9ff7d5 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.852595680181887, "train": {"Classification_binary": 2.7124409675598145, "Classification": 0.01975058577954769, "Regression_pt": 0.15982061624526978, "Regression_eta": 0.015520590357482433, "Regression_sin_phi": 0.008224372752010822, "Regression_cos_phi": 0.006810396444052458, "Regression_energy": 0.16151262819766998, "Total": 9.077138900756836, "Total_unweighted": 3.0840801573358476}, "train_group": {}, "valid": {"Classification_binary": 2.664217472076416, "Classification": 0.019261735181013744, "Regression_pt": 0.14232587814331055, "Regression_eta": 0.012233768900235495, "Regression_sin_phi": 0.006851806615789731, "Regression_cos_phi": 0.007209877793987592, "Regression_energy": 0.14740915099779764, "Total": 8.421727498372396, "Total_unweighted": 2.999509689708551, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01916489750146866, "diagnostic/loss/cld_pf/Classification_binary": 2.6721105575561523, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006942945998162031, "diagnostic/loss/cld_pf/Regression_energy": 0.1454780250787735, "diagnostic/loss/cld_pf/Regression_eta": 0.011275469325482845, "diagnostic/loss/cld_pf/Regression_pt": 0.14189493656158447, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006312543526291847, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08878105133771896, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19733166694641113, "diagnostic/regression/cld_pf/energy_residual_mean": 0.061661772429943085, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4346500039100647, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.044096771627664566, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19387049973011017, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07126391679048538, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4304724335670471, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9402046799659729, "metrics/particle/class_1/pid_accuracy": 0.9841412305831909, "metrics/particle/class_2/efficiency": 0.4421784281730652, "metrics/particle/class_2/pid_accuracy": 0.7446534633636475, "metrics/particle/class_3/efficiency": 0.7670314908027649, "metrics/particle/class_3/pid_accuracy": 0.9815202355384827, "metrics/particle/class_4/efficiency": 0.8365287780761719, "metrics/particle/class_4/pid_accuracy": 0.30036187171936035, "metrics/particle/class_5/efficiency": 0.9708737730979919, "metrics/particle/class_5/pid_accuracy": 0.6759999990463257, "metrics/particle/count/bias_mean": 0.42352941632270813, "metrics/particle/count/mae": 3.247058868408203, "metrics/particle/count/prediction_mean": 70.9888916015625, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05301470309495926, "metrics/particle/event/energy_response_mean": 1.0018061399459839, "metrics/particle/event/met_abs_error": 5.5398430824279785, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046302203088998795, "metrics/particle/event/scalar_pt_response_mean": 1.0146881341934204, "metrics/particle/event/vector_pt_closure": 0.04878121241927147, "metrics/particle/matched/delta_eta_abs_mean": 0.013399863615632057, "metrics/particle/matched/delta_phi_abs_mean": 0.01061388198286295, "metrics/particle/matched/delta_r_mean": 0.019181201234459877, "metrics/particle/matched/energy_relative_abs_mean": 0.08865800499916077, "metrics/particle/matched/pid_accuracy": 0.964607298374176, "metrics/particle/matched/pt_relative_abs_mean": 0.0844564363360405, "metrics/particle/matching/duplicate_fraction": 0.03618351370096207, "metrics/particle/matching/efficiency": 0.8259250521659851, "metrics/particle/matching/f1": 0.8234539031982422, "metrics/particle/matching/purity": 0.8209974765777588, "metrics/particle/matching/target_coverage_dr0p05": 0.7848098874092102, "metrics/particle/matching/target_coverage_dr0p10": 0.8616310954093933, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7238641977310181, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8656138777732849, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02319280430674553, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08321157842874527, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08173774182796478, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018003743141889572, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13374024629592896, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005388195160776377, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3783950209617615, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05810147896409035, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5869165658950806, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8166.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27880311012268066, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0397613048553467, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.147544026374817, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0381338596343994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1253661960363388, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.807889938354492, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014647943899035454, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7656759023666382, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.047630906105041504, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9279685020446777, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9899.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.805563747882843, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.3609411716461182, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.902976036071777, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3558216094970703, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.14974559843540192, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.03514663875102997, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.858380317687988, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.44117796421051025, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.960259437561035, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0343639999628067, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09131157398223877, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09336455911397934, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.020147070288658142, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1452721506357193, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0186542309820652, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8909761905670166, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20243772864341736, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.796189785003662, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1501.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8294228911399841, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8722641468048096, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5863.90869140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5863.73486328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28178295493125916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 297659.71875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07906977087259293, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9351930618286133, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.18546558916568756, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.743762016296387, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2580.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5460167527198792, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.500497341156006, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.665122032165527, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2394064664840698, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12551727890968323, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09253459423780441, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.480965614318848, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1960630416870117, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.427457809448242, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016396615654230118, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09201555699110031, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0860329419374466, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01234499178826809, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1415492296218872, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005541840801015496, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11492781341075897, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.003411782206967473, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5133923888206482, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19849.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15265072882175446, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7742486000061035, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6376268267631531, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.522580087184906, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09626449644565582, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.262656211853027, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003496503457427025, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2696516811847687, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02274169586598873, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5313681364059448, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22594.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8681712746620178, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.7530131340026855, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.674297332763672, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.4363322257995605, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.17635369300842285, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.07925811409950256, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.489011287689209, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8333845138549805, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.182730674743652, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019190581515431404, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08954401314258575, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08521745353937149, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014307326637208462, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1396271288394928, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028120342176407576, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22728455066680908, "metrics/particle/momentum/all/accepted/pz_bias": -0.024074915796518326, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.127643346786499, "metrics/particle/momentum/all/accepted/samples": 29516.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23803992569446564, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.929957926273346, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 432.0887451171875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 431.9709777832031, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1181250512599945, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 80731.578125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01220311876386404, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6057290434837341, "metrics/particle/momentum/all/assigned/pz_bias": -0.01243616733700037, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0910067558288574, "metrics/particle/momentum/all/assigned/samples": 35073.0, "metrics/particle/momentum/all/efficiency": 0.8256454467773438, "metrics/particle/momentum/all/event/energy_bias": 1.6134567260742188, "metrics/particle/momentum/all/event/p3_error_mean": 15.202690124511719, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.232978820800781, "metrics/particle/momentum/all/event/px_bias": -0.0989091694355011, "metrics/particle/momentum/all/event/py_bias": 0.13664604723453522, "metrics/particle/momentum/all/event/pz_abs_mean": 10.885208129882812, "metrics/particle/momentum/all/event/pz_bias": 0.078499436378479, "metrics/particle/momentum/all/event/pz_residual_rms": 18.95010757446289, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010868449695408344, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04765051230788231, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04003298655152321, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005423962138593197, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08866569399833679, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012431625509634614, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.142621248960495, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.029726482927799225, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8204569816589355, "metrics/particle/momentum/class_1/accepted/samples": 16088.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0793856680393219, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2952083349227905, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3130938708782196, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2561052739620209, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04422403872013092, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.659472465515137, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005075845867395401, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3112834095954895, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01293861959129572, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2277095317840576, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9384588599205017, "metrics/particle/momentum/class_1/event/energy_bias": 1.5458300113677979, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.598041534423828, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.062110900878906, "metrics/particle/momentum/class_1/event/px_bias": 0.14345383644104004, "metrics/particle/momentum/class_1/event/py_bias": -0.3621402382850647, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.576363563537598, "metrics/particle/momentum/class_1/event/pz_bias": -0.1950070559978485, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.813876152038574, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03936639055609703, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19118382036685944, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1913384050130844, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04756573960185051, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23158474266529083, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04334121197462082, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7211707830429077, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1617448776960373, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.1333842277526855, "metrics/particle/momentum/class_2/accepted/samples": 1269.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8798047304153442, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8078666925430298, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5715.12060546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5714.65771484375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3767951726913452, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 293923.96875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08087679743766785, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0882396697998047, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.052599772810935974, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.279041767120361, "metrics/particle/momentum/class_2/assigned/samples": 2646.0, "metrics/particle/momentum/class_2/efficiency": 0.43910035490989685, "metrics/particle/momentum/class_2/event/energy_bias": -4.417718410491943, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.553526878356934, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.630340099334717, "metrics/particle/momentum/class_2/event/px_bias": 0.31939369440078735, "metrics/particle/momentum/class_2/event/py_bias": 0.2632945775985718, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.476249694824219, "metrics/particle/momentum/class_2/event/pz_bias": -1.343908429145813, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.460070610046387, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02869265340268612, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13469603657722473, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13564972579479218, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03205419331789017, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17789597809314728, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006838775589130819, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17593412101268768, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0019937988836318254, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6116833090782166, "metrics/particle/momentum/class_3/accepted/samples": 11698.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31057822704315186, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6755404472351074, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7874165773391724, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.658739447593689, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15819783508777618, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.808218002319336, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008468834683299065, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5056098699569702, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0071369074285030365, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7057666778564453, "metrics/particle/momentum/class_3/assigned/samples": 14760.0, "metrics/particle/momentum/class_3/efficiency": 0.7701626420021057, "metrics/particle/momentum/class_3/event/energy_bias": 5.889138221740723, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.709722518920898, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.3849077224731445, "metrics/particle/momentum/class_3/event/px_bias": -0.4883871078491211, "metrics/particle/momentum/class_3/event/py_bias": 0.06603743135929108, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.862679481506348, "metrics/particle/momentum/class_3/event/pz_bias": 1.1739226579666138, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.35224723815918, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015240231528878212, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18228937685489655, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12561030685901642, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06686769425868988, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18144261837005615, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.49985432624816895, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.010688992217183113, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4360617399215698, "metrics/particle/momentum/class_4/accepted/samples": 294.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.19568336009979248, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9185309410095215, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9289273023605347, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.865180253982544, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.737606048583984, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0028248587623238564, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6052308082580566, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.04637571796774864, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7046540975570679, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8305084705352783, "metrics/particle/momentum/class_4/event/energy_bias": -1.4974812269210815, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0102086067199707, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5287322998046875, "metrics/particle/momentum/class_4/event/px_bias": -0.19743259251117706, "metrics/particle/momentum/class_4/event/py_bias": 0.19057554006576538, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0600193738937378, "metrics/particle/momentum/class_4/event/pz_bias": 0.47169196605682373, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5310683250427246, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008948228321969509, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02694721519947052, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027899308130145073, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00458802143111825, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07796125113964081, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14874249696731567, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.041439369320869446, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4796927273273468, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03892386704683304, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05081753432750702, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0510399155318737, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008653865195810795, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.20802775025367737, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.3514542579650879, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.07917343825101852, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.0852365493774414, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.09368860721588135, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8010044097900391, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6403930187225342, "metrics/particle/momentum/class_5/event/px_bias": 0.12406302243471146, "metrics/particle/momentum/class_5/event/py_bias": -0.02112129144370556, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.38325583934783936, "metrics/particle/momentum/class_5/event/pz_bias": -0.02819983661174774, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6635152101516724, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4862117767333984, "step_valid_time": 8013.610114097595, "step_total_time": 8015.096326589584, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0072254905676519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08764472036527726, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3049, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.95274040039383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9521154476877665, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524278215223098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04788455231223352, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9471611421069904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9465398491308625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9468503937007874, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.95274040039383, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007932627498029, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0903599207554664, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2156, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2150, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9464442493415277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9468599033816425, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466520307354556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05314009661835749, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9438103599648815, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9442248572683355, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9440175631174533, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9464442493415277, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0077111055503079, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08832891481061966, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1199, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9142640364188164, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9170471841704718, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9156534954407295, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08295281582952815, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.909711684370258, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9124809741248098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9110942249240122, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9142640364188164} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_19000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..c67bccde4b212201f221f9ed41c0558b7d23c994 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.8999621068586586, "train": {"Classification_binary": 2.7099297046661377, "Classification": 0.019923385232686996, "Regression_pt": 0.13712754845619202, "Regression_eta": 0.010742357932031155, "Regression_sin_phi": 0.006117274519056082, "Regression_cos_phi": 0.00740393903106451, "Regression_energy": 0.1420779675245285, "Total": 8.271320343017578, "Total_unweighted": 3.033322177361697}, "train_group": {}, "valid": {"Classification_binary": 2.6279446283976235, "Classification": 0.019061942895253498, "Regression_pt": 0.13979404171307883, "Regression_eta": 0.012082745631535849, "Regression_sin_phi": 0.006880574549237887, "Regression_cos_phi": 0.007203699400027593, "Regression_energy": 0.14504201213518778, "Total": 8.313844680786133, "Total_unweighted": 2.958009644721945, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01896853558719158, "diagnostic/loss/cld_pf/Classification_binary": 2.6232259273529053, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007108274381607771, "diagnostic/loss/cld_pf/Regression_energy": 0.14592061936855316, "diagnostic/loss/cld_pf/Regression_eta": 0.011280190199613571, "diagnostic/loss/cld_pf/Regression_pt": 0.1415240466594696, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006417613942176104, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09019722789525986, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19583193957805634, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06307794153690338, "diagnostic/regression/cld_pf/energy_residual_rms": 0.43244463205337524, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03529014065861702, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19022053480148315, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06245727464556694, "diagnostic/regression/cld_pf/pt_residual_rms": 0.42381760478019714, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9409521222114563, "metrics/particle/class_1/pid_accuracy": 0.9839094281196594, "metrics/particle/class_2/efficiency": 0.43192434310913086, "metrics/particle/class_2/pid_accuracy": 0.7296227812767029, "metrics/particle/class_3/efficiency": 0.7769866585731506, "metrics/particle/class_3/pid_accuracy": 0.9845570921897888, "metrics/particle/class_4/efficiency": 0.8314833641052246, "metrics/particle/class_4/pid_accuracy": 0.37742719054222107, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.6673306822776794, "metrics/particle/count/bias_mean": -0.040522877126932144, "metrics/particle/count/mae": 3.2705881595611572, "metrics/particle/count/prediction_mean": 70.52483367919922, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.051706112921237946, "metrics/particle/event/energy_response_mean": 0.9999330639839172, "metrics/particle/event/met_abs_error": 5.249253749847412, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04184864088892937, "metrics/particle/event/scalar_pt_response_mean": 1.0016543865203857, "metrics/particle/event/vector_pt_closure": 0.045983292162418365, "metrics/particle/matched/delta_eta_abs_mean": 0.011305385269224644, "metrics/particle/matched/delta_phi_abs_mean": 0.011064937338232994, "metrics/particle/matched/delta_r_mean": 0.017709964886307716, "metrics/particle/matched/energy_relative_abs_mean": 0.08894240111112595, "metrics/particle/matched/pid_accuracy": 0.9660488963127136, "metrics/particle/matched/pt_relative_abs_mean": 0.08307702094316483, "metrics/particle/matching/duplicate_fraction": 0.03292772173881531, "metrics/particle/matching/efficiency": 0.8296207189559937, "metrics/particle/matching/f1": 0.8298589587211609, "metrics/particle/matching/purity": 0.8300973773002625, "metrics/particle/matching/target_coverage_dr0p05": 0.8017505407333374, "metrics/particle/matching/target_coverage_dr0p10": 0.8636410236358643, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.728736162185669, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8668828010559082, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021129682660102844, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08598803728818893, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08146382123231888, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014938234351575375, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13375437259674072, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004839099943637848, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.36877527832984924, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.019482314586639404, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4939332008361816, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8266.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26175135374069214, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0619654655456543, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0595463514328003, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.94942706823349, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11829046159982681, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.00450325012207, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01650799997150898, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7706019878387451, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0212548915296793, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9429070949554443, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9874.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.815428614616394, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.2877403497695923, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.540401458740234, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3773889541625977, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2316000908613205, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.03910784423351288, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.488276481628418, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2824494242668152, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.045602798461914, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03210785239934921, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09434852004051208, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09094837307929993, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015534312464296818, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14302045106887817, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02107279747724533, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9312898516654968, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06345640867948532, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.383760929107666, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1566.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8189573287963867, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5425355434417725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 16952.998046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 16952.81640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27290448546409607, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 858441.0625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08226121217012405, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8954172134399414, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.020536456257104874, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.725536346435547, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2565.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5696616768836975, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.08371603488922119, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.594944000244141, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1499899625778198, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1151253804564476, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08691269159317017, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.437237739562988, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.805880069732666, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.45521354675293, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01528074685484171, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09136182069778442, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0847080647945404, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011568782851099968, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1394648551940918, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005547150503844023, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11925657838582993, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.014158686622977257, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5160490274429321, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19830.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.154433935880661, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6686168909072876, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.48867717385292053, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.37280282378196716, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09219353646039963, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.404650688171387, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027102679014205933, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2616536617279053, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.030049223452806473, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5607190132141113, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22507.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8673402667045593, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.5558270215988159, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.123063087463379, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.941754341125488, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4841104745864868, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9599735140800476, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2596349716186523, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.007864966988563538, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.59061336517334, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017799071967601776, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09002197533845901, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08413343876600266, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012717117555439472, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13808941841125488, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028319060802459717, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23166188597679138, "metrics/particle/momentum/all/accepted/pz_bias": 0.011544562876224518, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1851073503494263, "metrics/particle/momentum/all/accepted/samples": 29662.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23353181779384613, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8439023494720459, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1244.9466552734375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1244.827392578125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1128312274813652, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 232570.921875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012447776272892952, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5987725853919983, "metrics/particle/momentum/all/assigned/pz_bias": 0.02385144867002964, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0990512371063232, "metrics/particle/momentum/all/assigned/samples": 34946.0, "metrics/particle/momentum/all/efficiency": 0.8297294974327087, "metrics/particle/momentum/all/event/energy_bias": 0.6481972932815552, "metrics/particle/momentum/all/event/p3_error_mean": 14.97109603881836, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.72463846206665, "metrics/particle/momentum/all/event/px_bias": -0.3676357567310333, "metrics/particle/momentum/all/event/py_bias": -1.085994005203247, "metrics/particle/momentum/all/event/pz_abs_mean": 11.001482009887695, "metrics/particle/momentum/all/event/pz_bias": 2.0961945056915283, "metrics/particle/momentum/all/event/pz_residual_rms": 21.36531639099121, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011157927103340626, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04865274205803871, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03961379453539848, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006142565049231052, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08864010870456696, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013656961964443326, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14876320958137512, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.009696152061223984, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8051818609237671, "metrics/particle/momentum/class_1/accepted/samples": 16109.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07994689047336578, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.25884121656417847, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.29328617453575134, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23836283385753632, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.042953021824359894, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.403085708618164, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005252407398074865, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.30987903475761414, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.030738402158021927, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.453063488006592, "metrics/particle/momentum/class_1/assigned/samples": 17135.0, "metrics/particle/momentum/class_1/efficiency": 0.9396838545799255, "metrics/particle/momentum/class_1/event/energy_bias": 3.3032259941101074, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.852048873901367, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.8418731689453125, "metrics/particle/momentum/class_1/event/px_bias": 0.06721283495426178, "metrics/particle/momentum/class_1/event/py_bias": -0.07075515389442444, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.077019691467285, "metrics/particle/momentum/class_1/event/pz_bias": 0.5451563000679016, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.009428024291992, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03860022872686386, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19676543772220612, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19843436777591705, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0464089959859848, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2380838841199875, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04238618537783623, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7776755094528198, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08851775527000427, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.192858695983887, "metrics/particle/momentum/class_2/accepted/samples": 1274.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8977883458137512, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7412869930267334, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 16684.30078125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 16683.818359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36300843954086304, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 851661.4375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0798158124089241, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.017422914505005, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.009479086846113205, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.685336112976074, "metrics/particle/momentum/class_2/assigned/samples": 2606.0, "metrics/particle/momentum/class_2/efficiency": 0.44083043932914734, "metrics/particle/momentum/class_2/event/energy_bias": -8.609355926513672, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.867786407470703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.3724164962768555, "metrics/particle/momentum/class_2/event/px_bias": -0.7242878079414368, "metrics/particle/momentum/class_2/event/py_bias": 0.004404410719871521, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.822565078735352, "metrics/particle/momentum/class_2/event/pz_bias": -0.680120050907135, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.326043128967285, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02482612431049347, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13401362299919128, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13281068205833435, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02716316469013691, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17425976693630219, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005922666750848293, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1743059605360031, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.022692762315273285, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8438934683799744, "metrics/particle/momentum/class_3/accepted/samples": 11819.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.29787230491638184, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5159318447113037, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4231817722320557, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2909963130950928, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15000000596046448, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.579700469970703, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009059945121407509, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.510567307472229, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.016116391867399216, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7953267097473145, "metrics/particle/momentum/class_3/assigned/samples": 14680.0, "metrics/particle/momentum/class_3/efficiency": 0.7781289219856262, "metrics/particle/momentum/class_3/event/energy_bias": 6.822631359100342, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.232612609863281, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.5482707023620605, "metrics/particle/momentum/class_3/event/px_bias": 0.2955191135406494, "metrics/particle/momentum/class_3/event/py_bias": -1.1405867338180542, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.262969970703125, "metrics/particle/momentum/class_3/event/pz_bias": 1.7671359777450562, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.455854415893555, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013753841631114483, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16294929385185242, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1061462014913559, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.054112423211336136, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15806373953819275, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003436426166445017, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.40776029229164124, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04908961057662964, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.106493592262268, "metrics/particle/momentum/class_4/accepted/samples": 291.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.21574421226978302, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.451390027999878, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9574286937713623, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9020767211914062, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15625, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.400162696838379, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011363636702299118, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6013182997703552, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.09840882569551468, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8462212085723877, "metrics/particle/momentum/class_4/assigned/samples": 352.0, "metrics/particle/momentum/class_4/efficiency": 0.8220338821411133, "metrics/particle/momentum/class_4/event/energy_bias": -0.9427201747894287, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1256120204925537, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.587778091430664, "metrics/particle/momentum/class_4/event/px_bias": -0.1465989351272583, "metrics/particle/momentum/class_4/event/py_bias": 0.05740131437778473, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.12479829788208, "metrics/particle/momentum/class_4/event/pz_bias": 0.3132892847061157, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.369448661804199, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009550288319587708, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02651107870042324, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02393062599003315, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0035592406056821346, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06243828684091568, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18691515922546387, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.09775304049253464, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5197916030883789, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.016001293435692787, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04163912311196327, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03890511766076088, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0010080935899168253, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12043116241693497, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2585758864879608, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06288520991802216, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8877822756767273, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": 0.07441587001085281, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7891916632652283, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.622002899646759, "metrics/particle/momentum/class_5/event/px_bias": 0.14051900804042816, "metrics/particle/momentum/class_5/event/py_bias": 0.06354215741157532, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3821043074131012, "metrics/particle/momentum/class_5/event/pz_bias": 0.1507328748703003, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5615109205245972, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6474354267120361, "step_valid_time": 8460.9267411232, "step_total_time": 8462.57417678833, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013084760387292, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07891515435115004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2901, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2892, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520840170659666, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574257425742574, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954747408260655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04257425742574258, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9491302920905809, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9544554455445544, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.951785420437716, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520840170659666, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000960083166613, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08399289286457257, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2268, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2170, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2163, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9525899912203687, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9567901234567902, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9546854377474703, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043209876543209874, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9495171202809481, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9537037037037037, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9516058073031236, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9525899912203687, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047002748164886, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08263187046844112, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1307, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9279210925644916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9357306809487376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9318095238095238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06426931905126243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9349655700076511, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.931047619047619, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9279210925644916} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_2000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..7201d82b6049b0868aeda6eafc12bedc52fb5698 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.094732853353543, "train": {"Classification_binary": 3.872687578201294, "Classification": 0.032766859978437424, "Regression_pt": 0.2483806163072586, "Regression_eta": 0.01478066761046648, "Regression_sin_phi": 0.008318872191011906, "Regression_cos_phi": 0.00898655503988266, "Regression_energy": 0.2549203932285309, "Total": 13.049570083618164, "Total_unweighted": 4.440841542556882}, "train_group": {}, "valid": {"Classification_binary": 3.910323143005371, "Classification": 0.03068048010269801, "Regression_pt": 0.22695316871007284, "Regression_eta": 0.015848195801178615, "Regression_sin_phi": 0.008918635547161102, "Regression_cos_phi": 0.009339669098456701, "Regression_energy": 0.23228180408477783, "Total": 12.600629170735678, "Total_unweighted": 4.434345096349716, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.030628373846411705, "diagnostic/loss/cld_pf/Classification_binary": 3.893252372741699, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.009078411385416985, "diagnostic/loss/cld_pf/Regression_energy": 0.23499339818954468, "diagnostic/loss/cld_pf/Regression_eta": 0.014718860387802124, "diagnostic/loss/cld_pf/Regression_pt": 0.23100878298282623, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.008238768205046654, "diagnostic/regression/cld_pf/energy_pred_mean": 0.13866668939590454, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.25743743777275085, "diagnostic/regression/cld_pf/energy_residual_mean": 0.11154741048812866, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5327640175819397, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.08674456179141998, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.251557320356369, "diagnostic/regression/cld_pf/pt_residual_mean": 0.1139116957783699, "diagnostic/regression/cld_pf/pt_residual_rms": 0.5299485325813293, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.8981179594993591, "metrics/particle/class_1/pid_accuracy": 0.976697564125061, "metrics/particle/class_2/efficiency": 0.25202232599258423, "metrics/particle/class_2/pid_accuracy": 0.3987341821193695, "metrics/particle/class_3/efficiency": 0.707454264163971, "metrics/particle/class_3/pid_accuracy": 0.9772614240646362, "metrics/particle/class_4/efficiency": 0.8647830486297607, "metrics/particle/class_4/pid_accuracy": 0.11435239017009735, "metrics/particle/class_5/efficiency": 0.9611650705337524, "metrics/particle/class_5/pid_accuracy": 0.3414141535758972, "metrics/particle/count/bias_mean": -0.6215686202049255, "metrics/particle/count/mae": 4.722222328186035, "metrics/particle/count/prediction_mean": 69.94379425048828, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.10826811194419861, "metrics/particle/event/energy_response_mean": 1.0529111623764038, "metrics/particle/event/met_abs_error": 11.271215438842773, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09183341264724731, "metrics/particle/event/scalar_pt_response_mean": 1.0508146286010742, "metrics/particle/event/vector_pt_closure": 0.09084156155586243, "metrics/particle/matched/delta_eta_abs_mean": 0.019716694951057434, "metrics/particle/matched/delta_phi_abs_mean": 0.014499337412416935, "metrics/particle/matched/delta_r_mean": 0.02705085463821888, "metrics/particle/matched/energy_relative_abs_mean": 0.10921808332204819, "metrics/particle/matched/pid_accuracy": 0.9486915469169617, "metrics/particle/matched/pt_relative_abs_mean": 0.10032764077186584, "metrics/particle/matching/duplicate_fraction": 0.05046068876981735, "metrics/particle/matching/efficiency": 0.7652294635772705, "metrics/particle/matching/f1": 0.7686145901679993, "metrics/particle/matching/purity": 0.7720298171043396, "metrics/particle/matching/target_coverage_dr0p05": 0.6967072486877441, "metrics/particle/matching/target_coverage_dr0p10": 0.8076228499412537, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.653461754322052, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8085305690765381, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03263755142688751, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.112303726375103, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.09989321976900101, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.034054383635520935, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14783142507076263, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006082242354750633, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5259679555892944, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05276385694742203, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8429094552993774, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7563.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3942467272281647, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.6154656410217285, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7363108396530151, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6111754179000854, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1703878939151764, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.037995338439941, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02706039696931839, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2506895065307617, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08659282326698303, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.017026901245117, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9719.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7460787296295166, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 8.098556518554688, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.356073379516602, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.947030067443848, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.38656187057495117, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3701780140399933, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.494845390319824, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.802221417427063, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.36458969116211, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04287341609597206, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14108173549175262, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12251821160316467, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.05248264968395233, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16945438086986542, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021755438297986984, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3142814636230469, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1658419370651245, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.343397617340088, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1333.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9494675993919373, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.3067467212677, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 76162.921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 76162.78125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.34086325764656067, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3791863.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.10246066749095917, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.7968087196350098, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11773315072059631, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.472890853881836, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2479.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.48490360379219055, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 6.101417541503906, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.461214065551758, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.0057270526885986, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.19161950051784515, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07199586927890778, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.198410034179688, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 3.743140935897827, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 28.14235496520996, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023355528712272644, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10793357342481613, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.10153502970933914, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0318644717335701, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15354777872562408, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009739732486195862, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.15979963541030884, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.015096358023583889, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6604923605918884, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18481.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23076975345611572, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3005651235580444, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9681459665298462, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8446242809295654, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1363983005285263, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.663701057434082, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007938999682664871, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.513866662979126, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.040437981486320496, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 4.3802080154418945, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22295.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.808336615562439, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.020753860473633, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.614447593688965, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.890144348144531, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.1452983617782593, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.7164899110794067, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.653396129608154, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3843473196029663, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 21.786026000976562, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026870058849453926, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1107548400759697, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.10210316628217697, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03347335383296013, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15281085669994354, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033970121294260025, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3171673119068146, "metrics/particle/momentum/all/accepted/pz_bias": 0.03284202516078949, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3330475091934204, "metrics/particle/momentum/all/accepted/samples": 27377.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.32848480343818665, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4616076946258545, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5474.91845703125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5474.79248046875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16067028045654297, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1016542.25, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.02012002468109131, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.957423210144043, "metrics/particle/momentum/all/assigned/pz_bias": 0.05899808183312416, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.837072849273682, "metrics/particle/momentum/all/assigned/samples": 34493.0, "metrics/particle/momentum/all/efficiency": 0.7658116221427917, "metrics/particle/momentum/all/event/energy_bias": 19.220727920532227, "metrics/particle/momentum/all/event/p3_error_mean": 29.65670394897461, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.402871131896973, "metrics/particle/momentum/all/event/px_bias": -0.5671169757843018, "metrics/particle/momentum/all/event/py_bias": -2.158663749694824, "metrics/particle/momentum/all/event/pz_abs_mean": 21.878807067871094, "metrics/particle/momentum/all/event/pz_bias": 4.929709434509277, "metrics/particle/momentum/all/event/pz_residual_rms": 38.1272087097168, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019454900175333023, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.07629451155662537, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.06329552084207535, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.034072667360305786, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10923759639263153, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013696843525394797, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.24629974365234375, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.004601002670824528, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9346803426742554, "metrics/particle/momentum/class_1/accepted/samples": 15332.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14660657942295074, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5912247896194458, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.706187903881073, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.644192099571228, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08283264189958572, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 14.631659507751465, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010616578161716461, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5969306230545044, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02354300767183304, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.192070960998535, "metrics/particle/momentum/class_1/assigned/samples": 17143.0, "metrics/particle/momentum/class_1/efficiency": 0.8943592309951782, "metrics/particle/momentum/class_1/event/energy_bias": 15.958748817443848, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.05991554260254, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.197256088256836, "metrics/particle/momentum/class_1/event/px_bias": -0.8195182085037231, "metrics/particle/momentum/class_1/event/py_bias": -1.0979236364364624, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.522075653076172, "metrics/particle/momentum/class_1/event/pz_bias": 0.21205338835716248, "metrics/particle/momentum/class_1/event/pz_residual_rms": 28.797344207763672, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04755808413028717, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22273007035255432, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22249774634838104, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07256389409303665, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2606981098651886, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.07326478511095047, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.5720157623291016, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17421039938926697, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.474103927612305, "metrics/particle/momentum/class_2/accepted/samples": 778.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.2513030767440796, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.3054319620132446, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 79427.5, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 79426.9296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5014724731445312, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3872365.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.11611274629831314, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.4249444007873535, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07698214799165726, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.76645278930664, "metrics/particle/momentum/class_2/assigned/samples": 2377.0, "metrics/particle/momentum/class_2/efficiency": 0.26920413970947266, "metrics/particle/momentum/class_2/event/energy_bias": -8.631596565246582, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.383596420288086, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.81916618347168, "metrics/particle/momentum/class_2/event/px_bias": -0.732570230960846, "metrics/particle/momentum/class_2/event/py_bias": -1.5476818084716797, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.155492782592773, "metrics/particle/momentum/class_2/event/pz_bias": 0.9896098971366882, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.519981384277344, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.036232639104127884, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15114277601242065, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1492111086845398, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04006541147828102, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1910874843597412, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011109054321423173, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24253861606121063, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.10477696359157562, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9491561055183411, "metrics/particle/momentum/class_3/accepted/samples": 10802.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.39893510937690735, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5178768634796143, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.368156671524048, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2369158267974854, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19929392635822296, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.83536148071289, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01592136174440384, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8227328658103943, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.15889373421669006, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.3666672706604, "metrics/particle/momentum/class_3/assigned/samples": 14446.0, "metrics/particle/momentum/class_3/efficiency": 0.7111725807189941, "metrics/particle/momentum/class_3/event/energy_bias": 15.369266510009766, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.28139877319336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.503589630126953, "metrics/particle/momentum/class_3/event/px_bias": 0.5667862296104431, "metrics/particle/momentum/class_3/event/py_bias": 0.3195974826812744, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.661516189575195, "metrics/particle/momentum/class_3/event/pz_bias": 3.441567897796631, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.673547744750977, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022019965574145317, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16160187125205994, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11097078025341034, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04442497715353966, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16487109661102295, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009999999776482582, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7236357927322388, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.10348666459321976, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6429622173309326, "metrics/particle/momentum/class_4/accepted/samples": 300.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.19119957089424133, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.234055519104004, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.438185930252075, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.372262477874756, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12994350492954254, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.404125213623047, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8459550142288208, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.1121978834271431, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.076479911804199, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8474576473236084, "metrics/particle/momentum/class_4/event/energy_bias": -2.975740671157837, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.651047706604004, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.6019725799560547, "metrics/particle/momentum/class_4/event/px_bias": 0.2948380410671234, "metrics/particle/momentum/class_4/event/py_bias": 0.1810862421989441, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0911011695861816, "metrics/particle/momentum/class_4/event/pz_bias": 0.15518370270729065, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.0211992263793945, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014229674823582172, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.04835716634988785, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.04035375639796257, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.026293743401765823, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07265566289424896, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.41697582602500916, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.026906318962574005, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8327701687812805, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06994810700416565, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07370132952928543, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06517191231250763, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.012468848377466202, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15568508207798004, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5113104581832886, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.00016650540055707097, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2017543315887451, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": -0.49995070695877075, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8888684511184692, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3652064800262451, "metrics/particle/momentum/class_5/event/px_bias": 0.12334718555212021, "metrics/particle/momentum/class_5/event/py_bias": -0.013742052018642426, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.092397689819336, "metrics/particle/momentum/class_5/event/pz_bias": 0.13129493594169617, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.219728946685791, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4128210544586182, "step_valid_time": 894.7211050987244, "step_total_time": 896.1339263916016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.022501560353185, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18199439806526152, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2773, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2710, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9100754840827043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9106732348111658, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9103742613263296, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08932676518883416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8893994092550049, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8899835796387521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8896913985554826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9100754840827043, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.025182767857011, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19693992753627265, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2268, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2054, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2016, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9016681299385426, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9056437389770723, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9036515618125824, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09435626102292768, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8849868305531168, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8888888888888888, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8869335679718434, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9016681299385426, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0370464215794064, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19635152217172724, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1130, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8573596358118362, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8698999230177059, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8635842567825754, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13010007698229406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8277693474962063, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8398768283294842, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8337791364157432, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8573596358118362} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_20000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..2132bf0e84b9dd910e0ba89280474395e7ae0058 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9473285335354301, "train": {"Classification_binary": 2.722390651702881, "Classification": 0.019359134137630463, "Regression_pt": 0.15289674699306488, "Regression_eta": 0.020317815244197845, "Regression_sin_phi": 0.007927379570901394, "Regression_cos_phi": 0.008170614019036293, "Regression_energy": 0.1644625961780548, "Total": 9.331055641174316, "Total_unweighted": 3.0955249378457665}, "train_group": {}, "valid": {"Classification_binary": 2.6118127504984536, "Classification": 0.018757368127504986, "Regression_pt": 0.13949854175249735, "Regression_eta": 0.012188802162806192, "Regression_sin_phi": 0.0068533507486184435, "Regression_cos_phi": 0.007243244598309199, "Regression_energy": 0.14479168256123862, "Total": 8.283573150634766, "Total_unweighted": 2.9411457404494286, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01867292821407318, "diagnostic/loss/cld_pf/Classification_binary": 2.6052849292755127, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006947415415197611, "diagnostic/loss/cld_pf/Regression_energy": 0.14375969767570496, "diagnostic/loss/cld_pf/Regression_eta": 0.011126249097287655, "diagnostic/loss/cld_pf/Regression_pt": 0.1396586149930954, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00635403860360384, "diagnostic/regression/cld_pf/energy_pred_mean": 0.10732443630695343, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19898132979869843, "diagnostic/regression/cld_pf/energy_residual_mean": 0.08020515739917755, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4346555471420288, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.052143730223178864, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.19294576346874237, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07931086421012878, "diagnostic/regression/cld_pf/pt_residual_rms": 0.426136314868927, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9403196573257446, "metrics/particle/class_1/pid_accuracy": 0.984897255897522, "metrics/particle/class_2/efficiency": 0.42098668217658997, "metrics/particle/class_2/pid_accuracy": 0.7271988987922668, "metrics/particle/class_3/efficiency": 0.7543512582778931, "metrics/particle/class_3/pid_accuracy": 0.9845598340034485, "metrics/particle/class_4/efficiency": 0.8425832390785217, "metrics/particle/class_4/pid_accuracy": 0.371257483959198, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6487026214599609, "metrics/particle/count/bias_mean": -1.5385620594024658, "metrics/particle/count/mae": 3.4836602210998535, "metrics/particle/count/prediction_mean": 69.02679443359375, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05575260519981384, "metrics/particle/event/energy_response_mean": 0.997569739818573, "metrics/particle/event/met_abs_error": 5.335099220275879, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04353455826640129, "metrics/particle/event/scalar_pt_response_mean": 0.99995356798172, "metrics/particle/event/vector_pt_closure": 0.047827232629060745, "metrics/particle/matched/delta_eta_abs_mean": 0.01044556312263012, "metrics/particle/matched/delta_phi_abs_mean": 0.010644542053341866, "metrics/particle/matched/delta_r_mean": 0.016689889132976532, "metrics/particle/matched/energy_relative_abs_mean": 0.09030831605195999, "metrics/particle/matched/pid_accuracy": 0.9662976264953613, "metrics/particle/matched/pt_relative_abs_mean": 0.08387019485235214, "metrics/particle/matching/duplicate_fraction": 0.031133120879530907, "metrics/particle/matching/efficiency": 0.8189783692359924, "metrics/particle/matching/f1": 0.8280050158500671, "metrics/particle/matching/purity": 0.8372328877449036, "metrics/particle/matching/target_coverage_dr0p05": 0.7941184639930725, "metrics/particle/matching/target_coverage_dr0p10": 0.8546380996704102, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7158153057098389, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8554531335830688, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019242247566580772, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08732426166534424, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08181613683700562, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02449563890695572, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13549695909023285, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005532333627343178, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3726610243320465, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.018826233223080635, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5097306966781616, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8134.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26092854142189026, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9696096777915955, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9505927562713623, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8504292964935303, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11771982163190842, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.880982398986816, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016493145376443863, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7615891695022583, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.029415912926197052, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.83417010307312, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9701.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8024070262908936, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.08690191805362701, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.473121643066406, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.267735242843628, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1800651103258133, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.008594650775194168, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.473294258117676, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.030937224626541138, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.421367645263672, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028609273955225945, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09501846879720688, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09080235660076141, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021879777312278748, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14304842054843903, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017786560580134392, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9186238050460815, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.14491498470306396, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.283125162124634, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1518.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7444013953208923, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5502853393554688, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17205.373046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17205.21875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2667468786239624, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 858925.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08263136446475983, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.994924545288086, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.017622575163841248, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.962929725646973, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2493.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5522007942199707, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.4113735556602478, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.684467315673828, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1214561462402344, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.045131176710128784, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.04281219467520714, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.514780521392822, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.4704495668411255, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.357887268066406, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01446722261607647, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09256844222545624, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08553048223257065, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0224809180945158, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14052222669124603, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005600814474746585, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11727118492126465, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01742457039654255, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5068172812461853, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19640.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1485968381166458, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6510514616966248, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5196889042854309, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.41701170802116394, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09520821273326874, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.602878570556641, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028687973972409964, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2818112373352051, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.04193305969238281, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.4574224948883057, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22309.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8590298891067505, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.42209941148757935, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.403928756713867, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.133529186248779, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5124354958534241, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5537616610527039, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.4888968467712402, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3113176226615906, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.555769920349121, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01652606762945652, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09123916923999786, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0847722664475441, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023009223863482475, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13927830755710602, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028335382230579853, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22971807420253754, "metrics/particle/momentum/all/accepted/pz_bias": 0.009400865994393826, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.16779625415802, "metrics/particle/momentum/all/accepted/samples": 29292.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22323007881641388, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8055923581123352, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1243.7705078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1243.6646728515625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11393212527036667, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 230880.96875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012462683953344822, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6127424240112305, "metrics/particle/momentum/all/assigned/pz_bias": 0.03411052003502846, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4594249725341797, "metrics/particle/momentum/all/assigned/samples": 34503.0, "metrics/particle/momentum/all/efficiency": 0.8193795680999756, "metrics/particle/momentum/all/event/energy_bias": -0.09762781858444214, "metrics/particle/momentum/all/event/p3_error_mean": 15.399459838867188, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.867865085601807, "metrics/particle/momentum/all/event/px_bias": -0.2872392237186432, "metrics/particle/momentum/all/event/py_bias": -0.6051684617996216, "metrics/particle/momentum/all/event/pz_abs_mean": 11.348742485046387, "metrics/particle/momentum/all/event/pz_bias": 1.81270432472229, "metrics/particle/momentum/all/event/pz_residual_rms": 22.299671173095703, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009857557713985443, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04871191084384918, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03946242853999138, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005442603956907988, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08831606060266495, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001553277368657291, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14426231384277344, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.008421237580478191, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.888075053691864, "metrics/particle/momentum/class_1/accepted/samples": 16095.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07740017771720886, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19787921011447906, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.24730166792869568, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19041508436203003, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04395027086138725, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.636735439300537, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005661588627845049, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.30969393253326416, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.009596715681254864, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2794907093048096, "metrics/particle/momentum/class_1/assigned/samples": 17133.0, "metrics/particle/momentum/class_1/efficiency": 0.9388671517372131, "metrics/particle/momentum/class_1/event/energy_bias": 1.401326060295105, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.705574035644531, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.952974319458008, "metrics/particle/momentum/class_1/event/px_bias": -0.24077054858207703, "metrics/particle/momentum/class_1/event/py_bias": -0.27962052822113037, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.884843349456787, "metrics/particle/momentum/class_1/event/pz_bias": 0.2716790437698364, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.282955169677734, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03815450146794319, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19016380608081818, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1928142011165619, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04705438017845154, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2327316254377365, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0367346927523613, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6628390550613403, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13565824925899506, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8061602115631104, "metrics/particle/momentum/class_2/accepted/samples": 1225.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8471419811248779, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8208996653556824, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17245.123046875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17244.677734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3671089708805084, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 859960.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08283071964979172, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1063640117645264, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.007255129516124725, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.2919440269470215, "metrics/particle/momentum/class_2/assigned/samples": 2487.0, "metrics/particle/momentum/class_2/efficiency": 0.4238754212856293, "metrics/particle/momentum/class_2/event/energy_bias": -8.641743659973145, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.39616584777832, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.258199214935303, "metrics/particle/momentum/class_2/event/px_bias": -0.13582159578800201, "metrics/particle/momentum/class_2/event/py_bias": 0.11870887130498886, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.65040397644043, "metrics/particle/momentum/class_2/event/pz_bias": -0.9158152341842651, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.59017562866211, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023750556632876396, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1390191614627838, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13678623735904694, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.054630156606435776, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17869417369365692, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007819287711754441, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18934272229671478, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.019486520439386368, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8348532319068909, "metrics/particle/momentum/class_3/accepted/samples": 11510.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2915648818016052, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5081089735031128, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4277067184448242, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.320372462272644, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15400153398513794, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.598003387451172, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008427944965660572, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5456196665763855, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06463009864091873, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.6326026916503906, "metrics/particle/momentum/class_3/assigned/samples": 14357.0, "metrics/particle/momentum/class_3/efficiency": 0.7577852606773376, "metrics/particle/momentum/class_3/event/energy_bias": 8.471771240234375, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.156064987182617, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.350862979888916, "metrics/particle/momentum/class_3/event/px_bias": 0.07029636204242706, "metrics/particle/momentum/class_3/event/py_bias": -0.6048542857170105, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.382004737854004, "metrics/particle/momentum/class_3/event/pz_bias": 1.7580411434173584, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.474130630493164, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01378148328512907, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17401370406150818, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1134466677904129, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05233262479305267, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16572953760623932, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013559321872889996, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5718165636062622, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.23632130026817322, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2135679721832275, "metrics/particle/momentum/class_4/accepted/samples": 295.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22661671042442322, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9927325248718262, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.277794361114502, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.209681749343872, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1388102024793625, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.79387092590332, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016997167840600014, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6789732575416565, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1794523000717163, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.424135684967041, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8333333134651184, "metrics/particle/momentum/class_4/event/energy_bias": -1.3437352180480957, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1183574199676514, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5813260078430176, "metrics/particle/momentum/class_4/event/px_bias": -0.11209239065647125, "metrics/particle/momentum/class_4/event/py_bias": 0.16240501403808594, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1477464437484741, "metrics/particle/momentum/class_4/event/pz_bias": 0.5950884819030762, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.049587726593018, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007487965282052755, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024935590103268623, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02351878024637699, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.001209861016832292, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06458158046007156, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1317467838525772, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.07189862430095673, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.48353272676467896, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.018355613574385643, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04713546857237816, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04459831491112709, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0004757256538141519, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14206022024154663, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21270355582237244, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.018556809052824974, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8602449297904968, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.014753111638128757, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.706265926361084, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5961189270019531, "metrics/particle/momentum/class_5/event/px_bias": 0.13114897906780243, "metrics/particle/momentum/class_5/event/py_bias": -0.001807594671845436, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.29896751046180725, "metrics/particle/momentum/class_5/event/pz_bias": 0.10371086746454239, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.3404873609542847, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5358319282531738, "step_valid_time": 8905.98533320427, "step_total_time": 8907.521165370941, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036567567289068, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08068796644252552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2907, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9586478503446012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9621212121212122, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9603813907611376, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03787878787878788, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9540531670495569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9575098814229249, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9557783988163735, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9586478503446012, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002358562126742, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08301950439538519, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2276, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2159, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9477611940298507, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9485940246045694, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9481774264382961, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05140597539543058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9429323968393327, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9437609841827768, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9433465085638999, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9477611940298507, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003741298846547, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08556088784550031, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9218512898330804, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9225512528473804, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9222011385199241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0774487471526196, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9165402124430956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9172361427486713, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.916888045540797, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9218512898330804} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_21000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..f1adc904f323dc0cdf60d614be4928c503917923 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9946949602122016, "train": {"Classification_binary": 2.6030049324035645, "Classification": 0.018017388880252838, "Regression_pt": 0.14880052208900452, "Regression_eta": 0.012912055477499962, "Regression_sin_phi": 0.009705045260488987, "Regression_cos_phi": 0.007623186334967613, "Regression_energy": 0.1540486216545105, "Total": 8.695882797241211, "Total_unweighted": 2.954111752100289}, "train_group": {}, "valid": {"Classification_binary": 2.586444854736328, "Classification": 0.01864437262217204, "Regression_pt": 0.13814643025398254, "Regression_eta": 0.011913341780503591, "Regression_sin_phi": 0.0067714788019657135, "Regression_cos_phi": 0.007169816642999649, "Regression_energy": 0.14352640509605408, "Total": 8.198999404907227, "Total_unweighted": 2.9126166999340057, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018576979637145996, "diagnostic/loss/cld_pf/Classification_binary": 2.588780164718628, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006961616687476635, "diagnostic/loss/cld_pf/Regression_energy": 0.1424659937620163, "diagnostic/loss/cld_pf/Regression_eta": 0.011111886240541935, "diagnostic/loss/cld_pf/Regression_pt": 0.1379326581954956, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0062758829444646835, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08343921601772308, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19458231329917908, "diagnostic/regression/cld_pf/energy_residual_mean": 0.056319933384656906, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42895692586898804, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.0328163206577301, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18981720507144928, "diagnostic/regression/cld_pf/pt_residual_mean": 0.05998345836997032, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4213092029094696, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9401088356971741, "metrics/particle/class_1/pid_accuracy": 0.9851996898651123, "metrics/particle/class_2/efficiency": 0.442748099565506, "metrics/particle/class_2/pid_accuracy": 0.759135365486145, "metrics/particle/class_3/efficiency": 0.7805027961730957, "metrics/particle/class_3/pid_accuracy": 0.984542191028595, "metrics/particle/class_4/efficiency": 0.8203834295272827, "metrics/particle/class_4/pid_accuracy": 0.35424354672431946, "metrics/particle/class_5/efficiency": 0.9650485515594482, "metrics/particle/class_5/pid_accuracy": 0.6438632011413574, "metrics/particle/count/bias_mean": -0.6163398623466492, "metrics/particle/count/mae": 3.3222222328186035, "metrics/particle/count/prediction_mean": 69.94902038574219, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05302981659770012, "metrics/particle/event/energy_response_mean": 0.9890334606170654, "metrics/particle/event/met_abs_error": 5.121118068695068, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04207608476281166, "metrics/particle/event/scalar_pt_response_mean": 0.9984477162361145, "metrics/particle/event/vector_pt_closure": 0.04737677052617073, "metrics/particle/matched/delta_eta_abs_mean": 0.009836495853960514, "metrics/particle/matched/delta_phi_abs_mean": 0.010205709375441074, "metrics/particle/matched/delta_r_mean": 0.015873504802584648, "metrics/particle/matched/energy_relative_abs_mean": 0.08819727599620819, "metrics/particle/matched/pid_accuracy": 0.9675486087799072, "metrics/particle/matched/pt_relative_abs_mean": 0.08342226594686508, "metrics/particle/matching/duplicate_fraction": 0.032544709742069244, "metrics/particle/matching/efficiency": 0.8314268589019775, "metrics/particle/matching/f1": 0.8350737690925598, "metrics/particle/matching/purity": 0.838752806186676, "metrics/particle/matching/target_coverage_dr0p05": 0.8112165927886963, "metrics/particle/matching/target_coverage_dr0p10": 0.8637428879737854, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7283101081848145, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8665400743484497, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017303330823779106, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08544191718101501, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08252935111522675, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013943732716143131, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13559094071388245, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005087209399789572, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3760454058647156, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.031831659376621246, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6057682037353516, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8256.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25104624032974243, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9758679866790771, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0284596681594849, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9191156029701233, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11140637844800949, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.752320289611816, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014206868596374989, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7153502106666565, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.023870600387454033, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6191415786743164, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9784.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8144421577453613, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.09960176050662994, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.410863399505615, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2744193077087402, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.12495727092027664, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2210150510072708, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.393224716186523, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5347182750701904, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.743958473205566, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026987407356500626, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09400749951601028, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09381299465894699, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017058691009879112, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1459900587797165, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020357804372906685, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9550449848175049, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2583406865596771, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.165891170501709, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1621.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7520830035209656, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6472028493881226, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 16057.94140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 16057.7822265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2567940056324005, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 808967.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07798346132040024, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.746784210205078, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19805161654949188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.17044448852539, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2539.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5896689891815186, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.560894727706909, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.905496597290039, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.040321707725525, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14769700169563293, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.007704876363277435, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.765554428100586, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.95407634973526, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.366373062133789, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01453627459704876, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09087196737527847, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08496569842100143, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008331396616995335, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1397554874420166, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005027904990129173, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11589720845222473, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.008107529953122139, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5003761649131775, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19889.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14657536149024963, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6350954174995422, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4898405373096466, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.37275058031082153, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0900057926774025, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.315174579620361, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027179967146366835, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2553671896457672, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.014622403308749199, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3582892417907715, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22443.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8699208498001099, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.4523937702178955, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.255146026611328, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.998207092285156, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.2791515588760376, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6436277031898499, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.451122283935547, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6039755344390869, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.828969955444336, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015981819480657578, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08953661471605301, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08477175235748291, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010363326407968998, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13896244764328003, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028556070756167173, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23375119268894196, "metrics/particle/momentum/all/accepted/pz_bias": -0.02831496112048626, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1951158046722412, "metrics/particle/momentum/all/accepted/samples": 29766.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2201969027519226, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8049125075340271, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1173.3350830078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1173.2171630859375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10820917040109634, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 218617.625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0114479660987854, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5667685866355896, "metrics/particle/momentum/all/assigned/pz_bias": -0.030621113255620003, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.827864646911621, "metrics/particle/momentum/all/assigned/samples": 34766.0, "metrics/particle/momentum/all/efficiency": 0.832638680934906, "metrics/particle/momentum/all/event/energy_bias": -2.913686513900757, "metrics/particle/momentum/all/event/p3_error_mean": 14.641243934631348, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.659921169281006, "metrics/particle/momentum/all/event/px_bias": -1.3018913269042969, "metrics/particle/momentum/all/event/py_bias": 0.8723475933074951, "metrics/particle/momentum/all/event/pz_abs_mean": 10.846226692199707, "metrics/particle/momentum/all/event/pz_bias": -2.0927700996398926, "metrics/particle/momentum/all/event/pz_residual_rms": 18.073671340942383, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00954427570104599, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04763805866241455, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.040212780237197876, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0070188576355576515, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0898599848151207, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013041048077866435, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14395934343338013, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0205389354377985, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8193364143371582, "metrics/particle/momentum/class_1/accepted/samples": 16103.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07854590564966202, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.207896426320076, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.24934746325016022, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19475644826889038, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04428238049149513, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.9553098678588867, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004900816828012466, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2894219160079956, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.013462881557643414, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1528983116149902, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9393338561058044, "metrics/particle/momentum/class_1/event/energy_bias": 1.5725884437561035, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.991270065307617, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.580208778381348, "metrics/particle/momentum/class_1/event/px_bias": -0.3354412913322449, "metrics/particle/momentum/class_1/event/py_bias": 0.2865923047065735, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.367450714111328, "metrics/particle/momentum/class_1/event/pz_bias": -0.679627001285553, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.079460144042969, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03731478005647659, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1988607794046402, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2003498077392578, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.045966677367687225, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2399158775806427, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.040427155792713165, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8270251750946045, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.27359530329704285, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.327625274658203, "metrics/particle/momentum/class_2/accepted/samples": 1311.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8266839385032654, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7491897344589233, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 16011.541015625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 16011.0888671875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35113903880119324, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 807854.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07384131848812103, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9787023067474365, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.21947374939918518, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.068780899047852, "metrics/particle/momentum/class_2/assigned/samples": 2546.0, "metrics/particle/momentum/class_2/efficiency": 0.45363321900367737, "metrics/particle/momentum/class_2/event/energy_bias": -7.3135151863098145, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.379095077514648, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.106450080871582, "metrics/particle/momentum/class_2/event/px_bias": -0.4621276259422302, "metrics/particle/momentum/class_2/event/py_bias": 0.6351817846298218, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.711668014526367, "metrics/particle/momentum/class_2/event/pz_bias": -1.7336479425430298, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.517603874206543, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022534944117069244, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13301457464694977, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13232050836086273, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01984354853630066, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17393970489501953, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007563660619780421, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17583703994750977, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.015390259213745594, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7431495189666748, "metrics/particle/momentum/class_3/accepted/samples": 11899.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2811777889728546, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4602307081222534, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4783477783203125, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3422538042068481, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14038342237472534, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.210677146911621, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008177008479833603, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.47184765338897705, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.019900808110833168, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.185055732727051, "metrics/particle/momentum/class_3/assigned/samples": 14553.0, "metrics/particle/momentum/class_3/efficiency": 0.7833958864212036, "metrics/particle/momentum/class_3/event/energy_bias": 4.169414520263672, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.811929702758789, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.0546417236328125, "metrics/particle/momentum/class_3/event/px_bias": -0.5199008584022522, "metrics/particle/momentum/class_3/event/py_bias": -0.26905694603919983, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.107872009277344, "metrics/particle/momentum/class_3/event/pz_bias": -0.023190394043922424, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.457822799682617, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012878944166004658, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17838157713413239, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12332809716463089, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06643413752317429, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17624148726463318, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006968641187995672, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.44847771525382996, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12636180222034454, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3789249658584595, "metrics/particle/momentum/class_4/accepted/samples": 287.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3013938069343567, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.538891553878784, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.7849009037017822, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.719644784927368, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16949152946472168, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.959531784057617, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01977401040494442, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7221391797065735, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.03318813070654869, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.238320827484131, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8107344508171082, "metrics/particle/momentum/class_4/event/energy_bias": -1.353629231452942, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.158085823059082, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6016359329223633, "metrics/particle/momentum/class_4/event/px_bias": -0.04927036911249161, "metrics/particle/momentum/class_4/event/py_bias": 0.17100435495376587, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1889609098434448, "metrics/particle/momentum/class_4/event/pz_bias": 0.27593010663986206, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.790132999420166, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007615220732986927, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02041584625840187, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01948111690580845, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0031773648224771023, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04845709353685379, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1411624550819397, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.03938763216137886, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4568343460559845, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.032820235937833786, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05375008285045624, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04823894798755646, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.014034966006875038, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.034682080149650574, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.18117502331733704, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21603086590766907, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.016347922384738922, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.7598380446434021, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.011454805731773376, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7001079320907593, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5723569989204407, "metrics/particle/momentum/class_5/event/px_bias": 0.06484885513782501, "metrics/particle/momentum/class_5/event/py_bias": 0.04862614721059799, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.32024630904197693, "metrics/particle/momentum/class_5/event/pz_bias": 0.06776505708694458, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4539586305618286, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5887274742126465, "step_valid_time": 9350.153533935547, "step_total_time": 9351.742261648178, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999690743554169, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08088233237514542, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2905, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9570068920249426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9611074489123269, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9590527873704983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03889255108767304, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9533967837216935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574818721160184, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554349613550402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9570068920249426, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9981945604204563, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0853150540184926, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2166, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9572875385292823, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9558144647175203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04271246147071774, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508340649692713, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9537648612945839, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522972081776214, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543459174714662, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011772273471058, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07948845448726416, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1306, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9356814701378254, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9314024390243902, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06431852986217458, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9271623672230652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9356814701378254, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9314024390243902, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9271623672230652} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_22000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..1710a29ba6fe3e5bf6b4050681953ff79aea6403 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0420613868889732, "train": {"Classification_binary": 2.60073184967041, "Classification": 0.019209865480661392, "Regression_pt": 0.14427584409713745, "Regression_eta": 0.011449909768998623, "Regression_sin_phi": 0.006613713689148426, "Regression_cos_phi": 0.008269879035651684, "Regression_energy": 0.14879700541496277, "Total": 8.415474891662598, "Total_unweighted": 2.9393480671569705}, "train_group": {}, "valid": {"Classification_binary": 2.573053995768229, "Classification": 0.018642456581195194, "Regression_pt": 0.13503762086232504, "Regression_eta": 0.011803040901819864, "Regression_sin_phi": 0.006766278917590777, "Regression_cos_phi": 0.007201433181762695, "Regression_energy": 0.14030098915100098, "Total": 8.109373092651367, "Total_unweighted": 2.8928058153639236, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018561704084277153, "diagnostic/loss/cld_pf/Classification_binary": 2.573202133178711, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006778514478355646, "diagnostic/loss/cld_pf/Regression_energy": 0.13902050256729126, "diagnostic/loss/cld_pf/Regression_eta": 0.010880029760301113, "diagnostic/loss/cld_pf/Regression_pt": 0.13483425974845886, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006272576749324799, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08693653345108032, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19255584478378296, "diagnostic/regression/cld_pf/energy_residual_mean": 0.059817247092723846, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42679324746131897, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.039832763373851776, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18876579403877258, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0669998973608017, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4202837646007538, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.940798819065094, "metrics/particle/class_1/pid_accuracy": 0.9856383204460144, "metrics/particle/class_2/efficiency": 0.45425543189048767, "metrics/particle/class_2/pid_accuracy": 0.7481815814971924, "metrics/particle/class_3/efficiency": 0.7806346416473389, "metrics/particle/class_3/pid_accuracy": 0.9841506481170654, "metrics/particle/class_4/efficiency": 0.8284561038017273, "metrics/particle/class_4/pid_accuracy": 0.3897685706615448, "metrics/particle/class_5/efficiency": 0.9786407947540283, "metrics/particle/class_5/pid_accuracy": 0.5753968358039856, "metrics/particle/count/bias_mean": 0.008496732451021671, "metrics/particle/count/mae": 3.192810535430908, "metrics/particle/count/prediction_mean": 70.5738525390625, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.050805434584617615, "metrics/particle/event/energy_response_mean": 1.0011683702468872, "metrics/particle/event/met_abs_error": 5.334805965423584, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04168643802404404, "metrics/particle/event/scalar_pt_response_mean": 1.007279872894287, "metrics/particle/event/vector_pt_closure": 0.04589219391345978, "metrics/particle/matched/delta_eta_abs_mean": 0.010149926878511906, "metrics/particle/matched/delta_phi_abs_mean": 0.011155003681778908, "metrics/particle/matched/delta_r_mean": 0.016818450763821602, "metrics/particle/matched/energy_relative_abs_mean": 0.0872911661863327, "metrics/particle/matched/pid_accuracy": 0.9667826890945435, "metrics/particle/matched/pt_relative_abs_mean": 0.08341605961322784, "metrics/particle/matching/duplicate_fraction": 0.032071348279714584, "metrics/particle/matching/efficiency": 0.8328902721405029, "metrics/particle/matching/f1": 0.8328401446342468, "metrics/particle/matching/purity": 0.8327900171279907, "metrics/particle/matching/target_coverage_dr0p05": 0.8105126619338989, "metrics/particle/matching/target_coverage_dr0p10": 0.8672347664833069, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7295327186584473, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8688556551933289, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01834242045879364, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0839405208826065, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08169226348400116, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016052694991230965, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13451921939849854, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005558912176638842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3820767104625702, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.025744527578353882, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6041350364685059, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8275.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25730374455451965, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0307408571243286, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9384843707084656, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8322223424911499, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11543528735637665, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.552572250366211, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014188709668815136, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7544095516204834, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06577999889850616, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3527212142944336, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9867.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8163164854049683, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7246705889701843, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.968043804168701, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0300521850585938, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11578334122896194, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.16572734713554382, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.056058883666992, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3939637243747711, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.967536926269531, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025507695972919464, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09190410375595093, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09246108680963516, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01917809247970581, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14375850558280945, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022853612899780273, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9300320148468018, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.14105789363384247, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.194382429122925, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1619.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7437435984611511, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.383370041847229, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 13454.5546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 13454.3935546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2667188048362732, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 680188.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07899882644414902, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6891837120056152, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.27916479110717773, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.736464023590088, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2557.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5889414548873901, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.9206022620201111, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.362693786621094, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0403356552124023, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07060942053794861, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.030107393860816956, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.226254940032959, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.38405799865722656, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.95037269592285, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015573684126138687, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09051300585269928, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08556405454874039, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0139072947204113, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14080385863780975, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005518486723303795, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11388900130987167, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.017031891271471977, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.50595623254776, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19933.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14794215559959412, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6052305698394775, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5015267729759216, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3904036581516266, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.090614452958107, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.2068705558776855, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0030190020333975554, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2540303170681, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.015441450290381908, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4357740879058838, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22524.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8718453645706177, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.3655410408973694, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.717875003814697, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.640796661376953, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.1372387707233429, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.17931362986564636, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0693397521972656, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6675869226455688, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.794221878051758, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01688103750348091, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08876509219408035, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08486425876617432, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014788596890866756, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13925272226333618, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003151507116854191, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23259314894676208, "metrics/particle/momentum/all/accepted/pz_bias": 0.010868003591895103, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.199522852897644, "metrics/particle/momentum/all/accepted/samples": 29827.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22241094708442688, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7822996377944946, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 985.0020751953125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 984.8888549804688, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1105070412158966, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 183985.578125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011731715872883797, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5734742879867554, "metrics/particle/momentum/all/assigned/pz_bias": 0.008098606020212173, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.980172872543335, "metrics/particle/momentum/all/assigned/samples": 34948.0, "metrics/particle/momentum/all/efficiency": 0.8343450427055359, "metrics/particle/momentum/all/event/energy_bias": 0.16960936784744263, "metrics/particle/momentum/all/event/p3_error_mean": 15.019889831542969, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.655599117279053, "metrics/particle/momentum/all/event/px_bias": -0.049153998494148254, "metrics/particle/momentum/all/event/py_bias": 0.37514838576316833, "metrics/particle/momentum/all/event/pz_abs_mean": 11.102774620056152, "metrics/particle/momentum/all/event/pz_bias": 0.6774925589561462, "metrics/particle/momentum/all/event/pz_residual_rms": 20.68377685546875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008954182267189026, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04469009116292, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03846603259444237, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005755619611591101, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08637726306915283, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010563599644228816, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1342168152332306, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0056340014562010765, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7434247732162476, "metrics/particle/momentum/class_1/accepted/samples": 16093.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07858524471521378, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21707767248153687, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2544528543949127, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20146609842777252, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04527421295642853, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.210076808929443, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0053092180751264095, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.29673582315444946, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008427634835243225, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1317100524902344, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9387505054473877, "metrics/particle/momentum/class_1/event/energy_bias": 2.414216995239258, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.108589172363281, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.563961505889893, "metrics/particle/momentum/class_1/event/px_bias": -0.024693399667739868, "metrics/particle/momentum/class_1/event/py_bias": -0.013085395097732544, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.460354328155518, "metrics/particle/momentum/class_1/event/pz_bias": 0.40938231348991394, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.05426025390625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03588886931538582, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1963258534669876, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.196987122297287, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04277995601296425, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2370004504919052, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.046686746180057526, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8166992664337158, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11115898936986923, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.26439905166626, "metrics/particle/momentum/class_2/accepted/samples": 1328.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8432316184043884, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7601336240768433, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 13098.9794921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 13098.5302734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3591012954711914, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 671193.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07730388641357422, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.999894142150879, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.15464545786380768, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.999953269958496, "metrics/particle/momentum/class_2/assigned/samples": 2626.0, "metrics/particle/momentum/class_2/efficiency": 0.4595155715942383, "metrics/particle/momentum/class_2/event/energy_bias": -7.284746170043945, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.405696868896484, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.170135974884033, "metrics/particle/momentum/class_2/event/px_bias": -0.04975564777851105, "metrics/particle/momentum/class_2/event/py_bias": 0.47747161984443665, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.706657409667969, "metrics/particle/momentum/class_2/event/pz_bias": -1.9482165575027466, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.00005531311035, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0256976131349802, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13524779677391052, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13500380516052246, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.032445646822452545, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17711754143238068, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010041000787168741, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1825980395078659, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.026874033734202385, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8606665134429932, "metrics/particle/momentum/class_3/accepted/samples": 11951.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.28038790822029114, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4127579927444458, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.425313115119934, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2999293804168701, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1423404961824417, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.690743446350098, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007505970541387796, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4645031988620758, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03299334645271301, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6390955448150635, "metrics/particle/momentum/class_3/assigned/samples": 14655.0, "metrics/particle/momentum/class_3/efficiency": 0.7868193984031677, "metrics/particle/momentum/class_3/event/energy_bias": 6.412513732910156, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.418705940246582, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.08164119720459, "metrics/particle/momentum/class_3/event/px_bias": 0.011056721210479736, "metrics/particle/momentum/class_3/event/py_bias": -0.24199983477592468, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.766470432281494, "metrics/particle/momentum/class_3/event/pz_bias": 1.7468247413635254, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.509283065795898, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012321476824581623, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1655060052871704, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11644866317510605, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0583660751581192, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1687663495540619, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010489510372281075, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5485942363739014, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.19001318514347076, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8365477323532104, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2850552201271057, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.5789124965667725, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.89617121219635, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8309128284454346, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1525423675775528, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.000472068786621, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6858804821968079, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15807022154331207, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1928770542144775, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.2628947496414185, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9660766124725342, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4426745176315308, "metrics/particle/momentum/class_4/event/px_bias": -0.15270058810710907, "metrics/particle/momentum/class_4/event/py_bias": 0.14983397722244263, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0832395553588867, "metrics/particle/momentum/class_4/event/pz_bias": 0.36530959606170654, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.259213924407959, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006594982463866472, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023650316521525383, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022951921448111534, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004943396430462599, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06275781989097595, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15326060354709625, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03311098366975784, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5720877647399902, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.008937711827456951, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03516005352139473, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03345585986971855, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008352601900696754, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1000680923461914, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.16132529079914093, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.03008655086159706, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.5729455947875977, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": -0.10948042571544647, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8689934015274048, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7152791023254395, "metrics/particle/momentum/class_5/event/px_bias": 0.166938915848732, "metrics/particle/momentum/class_5/event/py_bias": 0.0029280055314302444, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.36821216344833374, "metrics/particle/momentum/class_5/event/pz_bias": 0.10419248044490814, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.599551796913147, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.667001485824585, "step_valid_time": 9795.537144422531, "step_total_time": 9797.204146623611, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0034382659511603, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07985062598187664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9618797239566218, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.961247947454844, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03812027604337825, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9576076240552087, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9569786535303777, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9606170003281916, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0051385542754045, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08087611188723098, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2179, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2173, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.956540825285338, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9586449626044875, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.957591738079543, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04135503739551254, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9539069359086918, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9560052793664761, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.954954954954955, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.956540825285338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006573608358596, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08187545794435558, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1229, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.93247344461305, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06752655538694992, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.93247344461305} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_23000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..c89f1ddddb0cb05fcf3773a5a2f8155d8395eb3c --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.0894278135657447, "train": {"Classification_binary": 2.500413656234741, "Classification": 0.01799825206398964, "Regression_pt": 0.12529011070728302, "Regression_eta": 0.009250307455658913, "Regression_sin_phi": 0.006781925912946463, "Regression_cos_phi": 0.005613113287836313, "Regression_energy": 0.12935934960842133, "Total": 7.533736228942871, "Total_unweighted": 2.794706715270877}, "train_group": {}, "valid": {"Classification_binary": 2.559809843699137, "Classification": 0.018557031949361164, "Regression_pt": 0.1347466210524241, "Regression_eta": 0.011580614993969599, "Regression_sin_phi": 0.006821637352307637, "Regression_cos_phi": 0.007127595444520314, "Regression_energy": 0.14015873273213705, "Total": 8.074481964111328, "Total_unweighted": 2.878802077223857, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018594762310385704, "diagnostic/loss/cld_pf/Classification_binary": 2.565599203109741, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006814207416027784, "diagnostic/loss/cld_pf/Regression_energy": 0.1376419961452484, "diagnostic/loss/cld_pf/Regression_eta": 0.01075210701674223, "diagnostic/loss/cld_pf/Regression_pt": 0.13319435715675354, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006362804211676121, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09175179898738861, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19303609430789948, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06463251262903214, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4227413833141327, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03929658606648445, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18789243698120117, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06646373122930527, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4143926501274109, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9441718459129333, "metrics/particle/class_1/pid_accuracy": 0.9858520030975342, "metrics/particle/class_2/efficiency": 0.4421784281730652, "metrics/particle/class_2/pid_accuracy": 0.7382118105888367, "metrics/particle/class_3/efficiency": 0.775492250919342, "metrics/particle/class_3/pid_accuracy": 0.9841022491455078, "metrics/particle/class_4/efficiency": 0.8213925361633301, "metrics/particle/class_4/pid_accuracy": 0.3611793518066406, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6679920554161072, "metrics/particle/count/bias_mean": 0.04313725605607033, "metrics/particle/count/mae": 3.1947712898254395, "metrics/particle/count/prediction_mean": 70.6084976196289, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.050818752497434616, "metrics/particle/event/energy_response_mean": 1.000190258026123, "metrics/particle/event/met_abs_error": 5.183789253234863, "metrics/particle/event/scalar_pt_relative_abs_error": 0.041243117302656174, "metrics/particle/event/scalar_pt_response_mean": 1.004805326461792, "metrics/particle/event/vector_pt_closure": 0.04474096745252609, "metrics/particle/matched/delta_eta_abs_mean": 0.009211922064423561, "metrics/particle/matched/delta_phi_abs_mean": 0.01112202275544405, "metrics/particle/matched/delta_r_mean": 0.01613159105181694, "metrics/particle/matched/energy_relative_abs_mean": 0.08812077343463898, "metrics/particle/matched/pid_accuracy": 0.9670087099075317, "metrics/particle/matched/pt_relative_abs_mean": 0.08327247947454453, "metrics/particle/matching/duplicate_fraction": 0.03222223371267319, "metrics/particle/matching/efficiency": 0.8312971591949463, "metrics/particle/matching/f1": 0.8310431838035583, "metrics/particle/matching/purity": 0.83078932762146, "metrics/particle/matching/target_coverage_dr0p05": 0.8087435960769653, "metrics/particle/matching/target_coverage_dr0p10": 0.8668920397758484, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7290603518486023, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8675681948661804, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01832016184926033, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08369425684213638, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08053095638751984, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01613837108016014, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13302026689052582, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004726700019091368, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3736686110496521, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.022590940818190575, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5690537691116333, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8251.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25987380743026733, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2442622184753418, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1840286254882812, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0787808895111084, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12153690308332443, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.916908264160156, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015065723098814487, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7679656744003296, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.027299152687191963, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9662721157073975, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9890.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.813948929309845, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.1193690299987793, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.9729437828063965, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.071742296218872, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4786469638347626, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.022828375920653343, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.033481597900391, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.263592392206192, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.192763328552246, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027020765468478203, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09503979980945587, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09265077859163284, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017627865076065063, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14433148503303528, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019801979884505272, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9668985605239868, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.09247306734323502, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0248775482177734, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1616.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7431991100311279, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5438038110733032, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 31319.90625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 31319.7421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2620716392993927, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1586993.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0763239860534668, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8039510250091553, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.03615241497755051, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.557290077209473, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2568.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5878501534461975, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.10788962244987488, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.403543472290039, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0434455871582031, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03473865985870361, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.13661128282546997, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.252926349639893, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.7126461267471313, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.50315284729004, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014120176434516907, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09077876806259155, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08492037653923035, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015263400040566921, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1398736536502838, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005536540993489325, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11539773643016815, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0010313969105482101, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5171733498573303, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19868.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15048716962337494, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6574653387069702, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5007842779159546, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.38868582248687744, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09394127875566483, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.757378101348877, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0023951034527271986, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.25335660576820374, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005981005262583494, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5982275009155273, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22546.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8690023422241211, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.6504635214805603, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.650974273681641, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.585566520690918, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.2893468737602234, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.4076230823993683, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0632779598236084, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3934766352176666, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.879560470581055, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015986712649464607, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08904450386762619, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08412250131368637, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01563469134271145, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1382552981376648, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027576929423958063, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23334017395973206, "metrics/particle/momentum/all/accepted/pz_bias": 0.0005538778495974839, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1658161878585815, "metrics/particle/momentum/all/accepted/samples": 29735.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2248763144016266, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8882828950881958, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2298.380859375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2298.2666015625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11407268047332764, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 429846.9375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011398697271943092, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5858731865882874, "metrics/particle/momentum/all/assigned/pz_bias": 0.014217681251466274, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0827596187591553, "metrics/particle/momentum/all/assigned/samples": 35004.0, "metrics/particle/momentum/all/efficiency": 0.8317714929580688, "metrics/particle/momentum/all/event/energy_bias": 0.36101603507995605, "metrics/particle/momentum/all/event/p3_error_mean": 14.790905952453613, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.338341236114502, "metrics/particle/momentum/all/event/px_bias": 0.73325514793396, "metrics/particle/momentum/all/event/py_bias": 0.5670627355575562, "metrics/particle/momentum/all/event/pz_abs_mean": 11.091789245605469, "metrics/particle/momentum/all/event/pz_bias": 1.5827618837356567, "metrics/particle/momentum/all/event/pz_residual_rms": 20.986125946044922, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008789499290287495, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04681574925780296, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03977130353450775, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0035641281865537167, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08891480416059494, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018560910830274224, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15045098960399628, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.023617126047611237, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.959703266620636, "metrics/particle/momentum/class_1/accepted/samples": 16163.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0744723528623581, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20874665677547455, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2740544080734253, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21654385328292847, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.040485356003046036, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.120121955871582, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0047252364456653595, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2880711257457733, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.026083482429385185, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4103848934173584, "metrics/particle/momentum/class_1/assigned/samples": 17142.0, "metrics/particle/momentum/class_1/efficiency": 0.9428337812423706, "metrics/particle/momentum/class_1/event/energy_bias": 1.1546095609664917, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.098276138305664, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.6107072830200195, "metrics/particle/momentum/class_1/event/px_bias": 0.2794364094734192, "metrics/particle/momentum/class_1/event/py_bias": 0.23468539118766785, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.290736198425293, "metrics/particle/momentum/class_1/event/pz_bias": -1.250230312347412, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.05788803100586, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036344561725854874, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19225215911865234, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19304871559143066, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0384538508951664, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2340557724237442, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031441718339920044, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.708359956741333, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.05349808558821678, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.919466257095337, "metrics/particle/momentum/class_2/accepted/samples": 1304.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8361179232597351, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7936489582061768, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 30731.58203125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 30731.12109375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3740924596786499, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1572066.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07604126632213593, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9531710147857666, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13502900302410126, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.556529521942139, "metrics/particle/momentum/class_2/assigned/samples": 2617.0, "metrics/particle/momentum/class_2/efficiency": 0.4512110650539398, "metrics/particle/momentum/class_2/event/energy_bias": -6.590089321136475, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.240568161010742, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.117948532104492, "metrics/particle/momentum/class_2/event/px_bias": 0.09715113043785095, "metrics/particle/momentum/class_2/event/py_bias": 0.5465996265411377, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.583791732788086, "metrics/particle/momentum/class_2/event/pz_bias": 0.6591796278953552, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.39789867401123, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023813335224986076, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1347367912530899, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13314999639987946, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.037433743476867676, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17499232292175293, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009310987079516053, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18179501593112946, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.027820851653814316, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6624631881713867, "metrics/particle/momentum/class_3/accepted/samples": 11814.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.29343339800834656, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6621530055999756, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.536010980606079, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4152202606201172, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15341758728027344, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.552112579345703, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00788150541484356, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.513361394405365, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.035949673503637314, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.81907057762146, "metrics/particle/momentum/class_3/assigned/samples": 14718.0, "metrics/particle/momentum/class_3/efficiency": 0.7777997255325317, "metrics/particle/momentum/class_3/event/energy_bias": 7.276885032653809, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.614187240600586, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.292700290679932, "metrics/particle/momentum/class_3/event/px_bias": 0.24013766646385193, "metrics/particle/momentum/class_3/event/py_bias": -0.4377339482307434, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.789088249206543, "metrics/particle/momentum/class_3/event/pz_bias": 1.617618203163147, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.167726516723633, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012050759978592396, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15488123893737793, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10344615578651428, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05150815472006798, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15154892206192017, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.37595629692077637, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0024412095081061125, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9837896823883057, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2415653020143509, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.7323501110076904, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8455419540405273, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7874999046325684, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.25024700164795, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7034608721733093, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.19784869253635406, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1298327445983887, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.5812122821807861, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9624996185302734, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5265660285949707, "metrics/particle/momentum/class_4/event/px_bias": -0.06214653700590134, "metrics/particle/momentum/class_4/event/py_bias": 0.25233328342437744, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9803777933120728, "metrics/particle/momentum/class_4/event/pz_bias": 0.4828401803970337, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5113677978515625, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006723971571773291, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02548830769956112, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025023212656378746, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.002742965007200837, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06952118128538132, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1409507542848587, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.005607709288597107, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.44486910104751587, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01490131113678217, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04230102524161339, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04087181016802788, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00020332280837465078, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12876999378204346, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.21192064881324768, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04461170360445976, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8642039895057678, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.10082316398620605, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7519874572753906, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6343356966972351, "metrics/particle/momentum/class_5/event/px_bias": 0.1786765605211258, "metrics/particle/momentum/class_5/event/py_bias": -0.02882157266139984, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3007688522338867, "metrics/particle/momentum/class_5/event/pz_bias": 0.07335411012172699, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.382552981376648, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.7245213985443115, "step_valid_time": 10241.016548156738, "step_total_time": 10242.741069793701, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017909056603984, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0797150332987252, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625861503117821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9632183908045977, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9629021667760999, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0367816091954023, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596324253363965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9602627257799672, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9599474720945502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625861503117821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004321124840044, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08384383003493401, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2162, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952610794207986, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528198376124644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04738920579201404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9490781387181738, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9486616937253182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9488698705288567, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530289727831431, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0087487324596274, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07919597744445639, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1232, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1228, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347496206373292, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9326267978803936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9336870026525199, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06737320211960636, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9317147192716236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9295987887963664, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9306555513452065, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347496206373292} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_24000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..2bb314c542ac33f3541c658924c1fbb8be25010d --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.1367942402425162, "train": {"Classification_binary": 2.5152194499969482, "Classification": 0.020961759611964226, "Regression_pt": 0.13773563504219055, "Regression_eta": 0.013157532550394535, "Regression_sin_phi": 0.008753383532166481, "Regression_cos_phi": 0.007660936564207077, "Regression_energy": 0.1423945426940918, "Total": 8.438920021057129, "Total_unweighted": 2.845883239991963}, "train_group": {}, "valid": {"Classification_binary": 2.532395521799723, "Classification": 0.01834758495291074, "Regression_pt": 0.1338341236114502, "Regression_eta": 0.011755317449569702, "Regression_sin_phi": 0.006594475358724594, "Regression_cos_phi": 0.0070906418065230055, "Regression_energy": 0.13878823320070902, "Total": 8.003025690714518, "Total_unweighted": 2.8488058981796103, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01826545223593712, "diagnostic/loss/cld_pf/Classification_binary": 2.532230854034424, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006872312165796757, "diagnostic/loss/cld_pf/Regression_energy": 0.13702669739723206, "diagnostic/loss/cld_pf/Regression_eta": 0.010462790727615356, "diagnostic/loss/cld_pf/Regression_pt": 0.1331178843975067, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006037701386958361, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09529493749141693, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1919190138578415, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06817565113306046, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4246061146259308, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04636743664741516, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18809731304645538, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07353457063436508, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41778799891471863, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9452259540557861, "metrics/particle/class_1/pid_accuracy": 0.9852595329284668, "metrics/particle/class_2/efficiency": 0.44559645652770996, "metrics/particle/class_2/pid_accuracy": 0.7686013579368591, "metrics/particle/class_3/efficiency": 0.7696466445922852, "metrics/particle/class_3/pid_accuracy": 0.9835531711578369, "metrics/particle/class_4/efficiency": 0.847628653049469, "metrics/particle/class_4/pid_accuracy": 0.3988095223903656, "metrics/particle/class_5/efficiency": 0.9825242757797241, "metrics/particle/class_5/pid_accuracy": 0.6897233128547668, "metrics/particle/count/bias_mean": -0.7699346542358398, "metrics/particle/count/mae": 3.2352941036224365, "metrics/particle/count/prediction_mean": 69.79542541503906, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04931192472577095, "metrics/particle/event/energy_response_mean": 0.9919455647468567, "metrics/particle/event/met_abs_error": 4.864925861358643, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039162710309028625, "metrics/particle/event/scalar_pt_response_mean": 1.0013452768325806, "metrics/particle/event/vector_pt_closure": 0.044538795948028564, "metrics/particle/matched/delta_eta_abs_mean": 0.00930212065577507, "metrics/particle/matched/delta_phi_abs_mean": 0.009509533643722534, "metrics/particle/matched/delta_r_mean": 0.014957144856452942, "metrics/particle/matched/energy_relative_abs_mean": 0.08757635951042175, "metrics/particle/matched/pid_accuracy": 0.9679684042930603, "metrics/particle/matched/pt_relative_abs_mean": 0.08372955769300461, "metrics/particle/matching/duplicate_fraction": 0.030996281653642654, "metrics/particle/matching/efficiency": 0.8298892974853516, "metrics/particle/matching/f1": 0.834441602230072, "metrics/particle/matching/purity": 0.8390440940856934, "metrics/particle/matching/target_coverage_dr0p05": 0.8130598068237305, "metrics/particle/matching/target_coverage_dr0p10": 0.8656694293022156, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7243643999099731, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.864270806312561, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017465511336922646, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0844905823469162, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08254218101501465, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02520989626646042, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1357591450214386, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0057045756839215755, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.36629718542099, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.022991139441728592, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.530562400817871, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8239.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24485886096954346, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8364181518554688, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8384448885917664, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7362858653068542, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1170831173658371, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.956252574920654, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015298317186534405, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7069187164306641, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.028142044320702553, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.619811773300171, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9805.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8127651214599609, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.16321823000907898, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.828388690948486, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.02394700050354, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.028333574533462524, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.3658701777458191, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.9042534828186035, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.4326069951057434, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.456515312194824, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027977269142866135, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09471965581178665, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09483661502599716, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.027299946174025536, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14636008441448212, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02469135820865631, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9728446006774902, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.03577585890889168, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7220065593719482, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1620.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.742199182510376, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3337969779968262, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5805.99755859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5805.83984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.261023610830307, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 292473.1875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07677165418863297, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.734203815460205, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.024041002616286278, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.57285213470459, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2540.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5893052220344543, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.8458878993988037, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.695940971374512, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9161988496780396, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.043612029403448105, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12484950572252274, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.572503566741943, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.9908788204193115, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.971352577209473, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01271086186170578, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08958686143159866, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08443490415811539, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017984068021178246, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13980069756507874, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006053574034012854, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11304767429828644, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.014628088101744652, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5284151434898376, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19823.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14177758991718292, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6468397378921509, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5163487792015076, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.41200268268585205, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09130260348320007, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.317910194396973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027667453978210688, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24836169183254242, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.033978499472141266, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.69320809841156, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22409.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8670340776443481, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.03849835693836212, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.707893371582031, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.622690677642822, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.27665480971336365, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.45437395572662354, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.093759059906006, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7151572108268738, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.42478084564209, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014863856136798859, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08845239132642746, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0844772458076477, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020498227328062057, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13905951380729675, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033353548496961594, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23027002811431885, "metrics/particle/momentum/all/accepted/pz_bias": 0.014198494143784046, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2620724439620972, "metrics/particle/momentum/all/accepted/samples": 29682.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21474134922027588, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7505310773849487, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 424.9015197753906, "metrics/particle/momentum/all/assigned/pt_relative_bias": 424.7937927246094, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11098003387451172, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 79067.953125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011710882186889648, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5594105124473572, "metrics/particle/momentum/all/assigned/pz_bias": 0.028091521933674812, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8242197036743164, "metrics/particle/momentum/all/assigned/samples": 34754.0, "metrics/particle/momentum/all/efficiency": 0.8302889466285706, "metrics/particle/momentum/all/event/energy_bias": -1.721168041229248, "metrics/particle/momentum/all/event/p3_error_mean": 13.913174629211426, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.294933319091797, "metrics/particle/momentum/all/event/px_bias": -0.3486004173755646, "metrics/particle/momentum/all/event/py_bias": 0.9450936317443848, "metrics/particle/momentum/all/event/pz_abs_mean": 10.170576095581055, "metrics/particle/momentum/all/event/pz_bias": 2.1386430263519287, "metrics/particle/momentum/all/event/pz_residual_rms": 18.514814376831055, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008758723735809326, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04474899172782898, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03787349537014961, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.001127691357396543, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08592941612005234, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016062272479757667, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13908010721206665, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.006251819897443056, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9044780135154724, "metrics/particle/momentum/class_1/accepted/samples": 16187.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06777440011501312, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2039298266172409, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2960311770439148, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24109722673892975, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03851990029215813, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.4357733726501465, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004435624927282333, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2627292275428772, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0017371210269629955, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.64362633228302, "metrics/particle/momentum/class_1/assigned/samples": 17134.0, "metrics/particle/momentum/class_1/efficiency": 0.944233775138855, "metrics/particle/momentum/class_1/event/energy_bias": 0.23427645862102509, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.963979721069336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.995190620422363, "metrics/particle/momentum/class_1/event/px_bias": -0.1230798065662384, "metrics/particle/momentum/class_1/event/py_bias": 0.11153502762317657, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.691323280334473, "metrics/particle/momentum/class_1/event/pz_bias": 0.40158727765083313, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.642600059509277, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03541228920221329, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19057030975818634, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19141671061515808, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.040350962430238724, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23216502368450165, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04633204638957977, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7228164672851562, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.030039992183446884, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.016125202178955, "metrics/particle/momentum/class_2/accepted/samples": 1295.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8728868961334229, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7848001718521118, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5770.1953125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5769.734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36242660880088806, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 291613.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08336594700813293, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0627496242523193, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.09571972489356995, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.8524370193481445, "metrics/particle/momentum/class_2/assigned/samples": 2555.0, "metrics/particle/momentum/class_2/efficiency": 0.4480968713760376, "metrics/particle/momentum/class_2/event/energy_bias": -7.887010097503662, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.019906044006348, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.115800380706787, "metrics/particle/momentum/class_2/event/px_bias": -0.031878337264060974, "metrics/particle/momentum/class_2/event/py_bias": 0.8049823045730591, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.3585844039917, "metrics/particle/momentum/class_2/event/pz_bias": -0.14970576763153076, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.105749130249023, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02120465412735939, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13647767901420593, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13722774386405945, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05344624072313309, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17903918027877808, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010225820587947965, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1869433969259262, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.026280170306563377, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0253815650939941, "metrics/particle/momentum/class_3/accepted/samples": 11735.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27449551224708557, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3658511638641357, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2789063453674316, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.169193148612976, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15263447165489197, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.141403198242188, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007772733457386494, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4716111123561859, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04251638427376747, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7344322204589844, "metrics/particle/momentum/class_3/assigned/samples": 14538.0, "metrics/particle/momentum/class_3/efficiency": 0.7725985646247864, "metrics/particle/momentum/class_3/event/energy_bias": 6.700355529785156, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.050912857055664, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.956491470336914, "metrics/particle/momentum/class_3/event/px_bias": -0.2348729372024536, "metrics/particle/momentum/class_3/event/py_bias": -0.31206953525543213, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.432567119598389, "metrics/particle/momentum/class_3/event/pz_bias": 1.2451283931732178, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.63516616821289, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012421280145645142, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16610027849674225, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10993354767560959, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.049382489174604416, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16272391378879547, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003389830468222499, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.45115289092063904, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11935587972402573, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5938361883163452, "metrics/particle/momentum/class_4/accepted/samples": 295.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2248346209526062, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.0410823822021484, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4318323135375977, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3669434785842896, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.878067970275879, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6544911861419678, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.19167670607566833, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.482381820678711, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8333333134651184, "metrics/particle/momentum/class_4/event/energy_bias": -1.1860113143920898, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1093456745147705, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5844848155975342, "metrics/particle/momentum/class_4/event/px_bias": -0.1057049036026001, "metrics/particle/momentum/class_4/event/py_bias": 0.22116771340370178, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1443513631820679, "metrics/particle/momentum/class_4/event/pz_bias": 0.3673354387283325, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.42480993270874, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006186410319060087, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021990563720464706, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02183864451944828, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003938185516744852, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06306672096252441, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15098123252391815, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.09138524532318115, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5988273024559021, "metrics/particle/momentum/class_5/accepted/samples": 170.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.00833059847354889, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.031040852889418602, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.02953161485493183, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.004367672372609377, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.011560693383216858, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.09225483983755112, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.15522348880767822, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.09253352880477905, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.5974318981170654, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9826589822769165, "metrics/particle/momentum/class_5/event/energy_bias": 0.4172217845916748, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9337917566299438, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.645960807800293, "metrics/particle/momentum/class_5/event/px_bias": 0.14693555235862732, "metrics/particle/momentum/class_5/event/py_bias": 0.11947816610336304, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5352480411529541, "metrics/particle/momentum/class_5/event/pz_bias": 0.2742977440357208, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.161146402359009, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5045700073242188, "step_valid_time": 10687.452286243439, "step_total_time": 10688.956856489182, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013564262863444, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07568235535036949, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3054, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2920, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2906, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9583196586806695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9561231172233137, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9572201278478938, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04387688277668631, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9537249753856252, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9515389652914211, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526307162760204, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9583196586806695, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045888276245716, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08377543930976517, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2171, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0456540825285338, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543459174714662, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0059884898057276, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0832175951659635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1230, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9332321699544764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9339407744874715, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9335863377609108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06605922551252848, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9309559939301972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9316628701594533, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9313092979127134, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9332321699544764} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_25000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..f417d34e051212d2208a373db86ee4caec5e44ce --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1841606669192877, "train": {"Classification_binary": 2.584371566772461, "Classification": 0.017111044377088547, "Regression_pt": 0.13982979953289032, "Regression_eta": 0.013334518298506737, "Regression_sin_phi": 0.006275337189435959, "Regression_cos_phi": 0.008435557596385479, "Regression_energy": 0.14696025848388672, "Total": 8.31123161315918, "Total_unweighted": 2.9163180822506547}, "train_group": {}, "valid": {"Classification_binary": 2.5309675534566245, "Classification": 0.018211313833793003, "Regression_pt": 0.13356377681096396, "Regression_eta": 0.011566019306580225, "Regression_sin_phi": 0.0066881732394297915, "Regression_cos_phi": 0.007013948634266853, "Regression_energy": 0.1385087271531423, "Total": 7.980185826619466, "Total_unweighted": 2.8465195124348006, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018100522458553314, "diagnostic/loss/cld_pf/Classification_binary": 2.538811206817627, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00677785649895668, "diagnostic/loss/cld_pf/Regression_energy": 0.13702431321144104, "diagnostic/loss/cld_pf/Regression_eta": 0.010763692669570446, "diagnostic/loss/cld_pf/Regression_pt": 0.13311967253684998, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0062829493544995785, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09118327498435974, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.192209392786026, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06406399607658386, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4233079254627228, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.044013891369104385, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18836839497089386, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0711810290813446, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41779279708862305, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9423511624336243, "metrics/particle/class_1/pid_accuracy": 0.9850518703460693, "metrics/particle/class_2/efficiency": 0.44867268204689026, "metrics/particle/class_2/pid_accuracy": 0.7585068345069885, "metrics/particle/class_3/efficiency": 0.7814258337020874, "metrics/particle/class_3/pid_accuracy": 0.9839980006217957, "metrics/particle/class_4/efficiency": 0.8082744479179382, "metrics/particle/class_4/pid_accuracy": 0.38576778769493103, "metrics/particle/class_5/efficiency": 0.9825242757797241, "metrics/particle/class_5/pid_accuracy": 0.6660078763961792, "metrics/particle/count/bias_mean": -0.3901960849761963, "metrics/particle/count/mae": 3.226797342300415, "metrics/particle/count/prediction_mean": 70.17516326904297, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05104507878422737, "metrics/particle/event/energy_response_mean": 0.9965829253196716, "metrics/particle/event/met_abs_error": 5.115163803100586, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04145871475338936, "metrics/particle/event/scalar_pt_response_mean": 1.0079686641693115, "metrics/particle/event/vector_pt_closure": 0.04422898590564728, "metrics/particle/matched/delta_eta_abs_mean": 0.010950138792395592, "metrics/particle/matched/delta_phi_abs_mean": 0.010467909276485443, "metrics/particle/matched/delta_r_mean": 0.016935911029577255, "metrics/particle/matched/energy_relative_abs_mean": 0.08806023001670837, "metrics/particle/matched/pid_accuracy": 0.9675902724266052, "metrics/particle/matched/pt_relative_abs_mean": 0.08358742296695709, "metrics/particle/matching/duplicate_fraction": 0.032523658126592636, "metrics/particle/matching/efficiency": 0.8333534002304077, "metrics/particle/matching/f1": 0.8356638550758362, "metrics/particle/matching/purity": 0.8379871249198914, "metrics/particle/matching/target_coverage_dr0p05": 0.8133839964866638, "metrics/particle/matching/target_coverage_dr0p10": 0.8680127859115601, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7307367920875549, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.867901623249054, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01913682371377945, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08383052796125412, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08215071260929108, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019304530695080757, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13471896946430206, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00579080730676651, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3763810992240906, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.023605072870850563, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5878239870071411, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8289.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2523357570171356, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9416831135749817, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0643783807754517, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9605852365493774, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1134629100561142, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.24601936340332, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013839421793818474, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7021106481552124, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06543093174695969, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5117335319519043, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9827.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8176975250244141, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.26370030641555786, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.933294296264648, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.054469347000122, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.284071147441864, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5488182306289673, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.9767537117004395, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.5442901849746704, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.866500854492188, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026866896077990532, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09052515029907227, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09122147411108017, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01923578791320324, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1420285403728485, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02216748706996441, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9376517534255981, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1051374226808548, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.010375738143921, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1624.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7273457050323486, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7121373414993286, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11036.2041015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11036.0458984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2615021765232086, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 556378.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07746756076812744, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7428290843963623, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.11207066476345062, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.719899654388428, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2543.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5907602906227112, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.5939435958862305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.703396797180176, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9375501275062561, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03373423218727112, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04034552723169327, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.5753302574157715, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6164394021034241, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.789887428283691, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015073630027472973, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09140335023403168, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08542903512716293, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018330087885260582, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13946926593780518, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006030453951098025, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11404465138912201, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0052261981181800365, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5096074342727661, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19899.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1433492749929428, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6472778916358948, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5210184454917908, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.41547486186027527, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08946269005537033, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.247041702270508, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0031998578924685717, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2606789171695709, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.004700681194663048, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7949293851852417, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22501.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8703582286834717, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.7440922260284424, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.63858699798584, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.56451416015625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.19563136994838715, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.579708456993103, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9983572959899902, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.08532407134771347, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.054279804229736, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01684580370783806, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08924993872642517, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.084833063185215, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018650364130735397, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13830682635307312, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032201798167079687, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23185104131698608, "metrics/particle/momentum/all/accepted/pz_bias": -0.01577894017100334, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1696199178695679, "metrics/particle/momentum/all/accepted/samples": 29812.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21665124595165253, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8079001307487488, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 805.4615478515625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 805.352783203125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10877232998609543, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 150248.890625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011614235118031502, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5660919547080994, "metrics/particle/momentum/all/assigned/pz_bias": -0.023578772321343422, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8640763759613037, "metrics/particle/momentum/all/assigned/samples": 34871.0, "metrics/particle/momentum/all/efficiency": 0.8339254260063171, "metrics/particle/momentum/all/event/energy_bias": 0.41384899616241455, "metrics/particle/momentum/all/event/p3_error_mean": 13.722546577453613, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.119100093841553, "metrics/particle/momentum/all/event/px_bias": 0.5134367346763611, "metrics/particle/momentum/all/event/py_bias": -1.0881812572479248, "metrics/particle/momentum/all/event/pz_abs_mean": 10.035006523132324, "metrics/particle/momentum/all/event/pz_bias": -2.0754055976867676, "metrics/particle/momentum/all/event/pz_residual_rms": 18.074705123901367, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009794575162231922, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.048327960073947906, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.040460411459207535, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011352639645338058, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08908940851688385, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001611004350706935, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14436937868595123, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.017213890329003334, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8783835172653198, "metrics/particle/momentum/class_1/accepted/samples": 16139.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07404022663831711, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19801324605941772, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21922527253627777, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1719215214252472, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.042004551738500595, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.5124354362487793, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004725512117147446, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2688625454902649, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.028802508488297462, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.5149723291397095, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.941433846950531, "metrics/particle/momentum/class_1/event/energy_bias": 4.051270484924316, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.365076065063477, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.310922145843506, "metrics/particle/momentum/class_1/event/px_bias": 0.3817005157470703, "metrics/particle/momentum/class_1/event/py_bias": -0.26421016454696655, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.816192626953125, "metrics/particle/momentum/class_1/event/pz_bias": -0.9328530430793762, "metrics/particle/momentum/class_1/event/pz_residual_rms": 12.254084587097168, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03618699312210083, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19408734142780304, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19639982283115387, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04260129854083061, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23643028736114502, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.043812453746795654, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7768858671188354, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.18854013085365295, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.111441135406494, "metrics/particle/momentum/class_2/accepted/samples": 1301.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8226375579833984, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7650852799415588, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10888.5537109375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10888.09765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3476911187171936, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 552695.9375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07605743408203125, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.916816234588623, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19907760620117188, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.613694667816162, "metrics/particle/momentum/class_2/assigned/samples": 2577.0, "metrics/particle/momentum/class_2/efficiency": 0.450173020362854, "metrics/particle/momentum/class_2/event/energy_bias": -7.1421661376953125, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.21971321105957, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.077692031860352, "metrics/particle/momentum/class_2/event/px_bias": -0.0073781609535217285, "metrics/particle/momentum/class_2/event/py_bias": -0.2501961290836334, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.595681190490723, "metrics/particle/momentum/class_2/event/pz_bias": -2.1892497539520264, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.29458999633789, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02449657954275608, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13227298855781555, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1328773945569992, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.034324876964092255, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17402292788028717, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008387151174247265, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17682217061519623, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00287804938852787, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.675684928894043, "metrics/particle/momentum/class_3/accepted/samples": 11923.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27862459421157837, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4897584915161133, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5675075054168701, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4458743333816528, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14398030936717987, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.947853088378906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008204006589949131, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5002527832984924, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.014755578711628914, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0009188652038574, "metrics/particle/momentum/class_3/assigned/samples": 14627.0, "metrics/particle/momentum/class_3/efficiency": 0.7849759459495544, "metrics/particle/momentum/class_3/event/energy_bias": 4.827922821044922, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.278803825378418, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.05258846282959, "metrics/particle/momentum/class_3/event/px_bias": -0.046824581921100616, "metrics/particle/momentum/class_3/event/py_bias": -0.6081331372261047, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.616517066955566, "metrics/particle/momentum/class_3/event/pz_bias": 0.7165642976760864, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.514591217041016, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012821785174310207, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16720716655254364, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11552669107913971, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06200455129146576, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16737119853496552, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010714286006987095, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5040166974067688, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07567939907312393, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7593693733215332, "metrics/particle/momentum/class_4/accepted/samples": 280.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24795886874198914, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.8568806648254395, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.131901264190674, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.067707061767578, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.18980169296264648, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.290505409240723, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01983002759516239, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.744053065776825, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05797009542584419, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.445178508758545, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.790960431098938, "metrics/particle/momentum/class_4/event/energy_bias": -1.3347550630569458, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0977251529693604, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5515127182006836, "metrics/particle/momentum/class_4/event/px_bias": -0.004312172532081604, "metrics/particle/momentum/class_4/event/py_bias": 0.09635131806135178, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1263951063156128, "metrics/particle/momentum/class_4/event/pz_bias": 0.26252683997154236, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.3511881828308105, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008227158337831497, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025667788460850716, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023028947412967682, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00941923726350069, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05592956393957138, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12344754487276077, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01657581701874733, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.41266727447509766, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01626601442694664, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04242568463087082, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03956273943185806, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.007036654278635979, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12675349414348602, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20310497283935547, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0627460703253746, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9275599122047424, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": 0.011577211320400238, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.6779971122741699, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5565260648727417, "metrics/particle/momentum/class_5/event/px_bias": 0.19025112688541412, "metrics/particle/momentum/class_5/event/py_bias": -0.061993032693862915, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.30379876494407654, "metrics/particle/momentum/class_5/event/pz_bias": 0.06760625541210175, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4362332820892334, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5850555896759033, "step_valid_time": 11134.328974485397, "step_total_time": 11135.914030313492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00561271138351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07889849415623129, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9501148670823761, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9554455445544554, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952772749712029, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04455445544554455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9458483754512635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9511551155115512, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9484943228566727, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9501148670823761, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005686062985391, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08293921373847224, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2178, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2168, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561018437225637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9582050153981522, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571522742254449, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04179498460184778, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.95171202809482, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9538055433347998, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9527576356844649, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561018437225637, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050805945482997, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0825726738036765, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9363153904473086, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9366704588547592, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06368460955269144, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9347990902198635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9351535836177475, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9370257966616085} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_26000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..5deefc5f32a7653586452ee1559b80cf1c5fee34 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2315270935960592, "train": {"Classification_binary": 2.4545371532440186, "Classification": 0.0206583421677351, "Regression_pt": 0.12472694367170334, "Regression_eta": 0.01062261313199997, "Regression_sin_phi": 0.006538952700793743, "Regression_cos_phi": 0.006844094023108482, "Regression_energy": 0.13351047039031982, "Total": 7.788784027099609, "Total_unweighted": 2.757438569329679}, "train_group": {}, "valid": {"Classification_binary": 2.4921773274739585, "Classification": 0.01803986355662346, "Regression_pt": 0.1305588185787201, "Regression_eta": 0.011284554998079935, "Regression_sin_phi": 0.00674613689382871, "Regression_cos_phi": 0.007081853225827217, "Regression_energy": 0.13552138209342957, "Total": 7.8592783610026045, "Total_unweighted": 2.8014099368204675, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018050458282232285, "diagnostic/loss/cld_pf/Classification_binary": 2.486117362976074, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0068578762002289295, "diagnostic/loss/cld_pf/Regression_energy": 0.13314466178417206, "diagnostic/loss/cld_pf/Regression_eta": 0.010483200661838055, "diagnostic/loss/cld_pf/Regression_pt": 0.12895944714546204, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0063281976617872715, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09096701443195343, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1904735416173935, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06384773552417755, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41992974281311035, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03580715134739876, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1846475750207901, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06297428905963898, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41156813502311707, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.943750262260437, "metrics/particle/class_1/pid_accuracy": 0.9857847690582275, "metrics/particle/class_2/efficiency": 0.4452546536922455, "metrics/particle/class_2/pid_accuracy": 0.7640736699104309, "metrics/particle/class_3/efficiency": 0.7796237468719482, "metrics/particle/class_3/pid_accuracy": 0.9855676889419556, "metrics/particle/class_4/efficiency": 0.8123108148574829, "metrics/particle/class_4/pid_accuracy": 0.39751553535461426, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6646706461906433, "metrics/particle/count/bias_mean": -0.15032680332660675, "metrics/particle/count/mae": 3.197385549545288, "metrics/particle/count/prediction_mean": 70.41503143310547, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.048663873225450516, "metrics/particle/event/energy_response_mean": 0.9968299865722656, "metrics/particle/event/met_abs_error": 4.8532209396362305, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03877102956175804, "metrics/particle/event/scalar_pt_response_mean": 1.0007399320602417, "metrics/particle/event/vector_pt_closure": 0.04292287304997444, "metrics/particle/matched/delta_eta_abs_mean": 0.009147314354777336, "metrics/particle/matched/delta_phi_abs_mean": 0.010538598522543907, "metrics/particle/matched/delta_r_mean": 0.015676580369472504, "metrics/particle/matched/energy_relative_abs_mean": 0.08835066854953766, "metrics/particle/matched/pid_accuracy": 0.9690102338790894, "metrics/particle/matched/pt_relative_abs_mean": 0.08213897049427032, "metrics/particle/matching/duplicate_fraction": 0.029256973415613174, "metrics/particle/matching/efficiency": 0.8329828977584839, "metrics/particle/matching/f1": 0.8338711261749268, "metrics/particle/matching/purity": 0.8347612023353577, "metrics/particle/matching/target_coverage_dr0p05": 0.8091325759887695, "metrics/particle/matching/target_coverage_dr0p10": 0.8665308356285095, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7332746982574463, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.869633674621582, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018446411937475204, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08531460165977478, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08021625131368637, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016559386625885963, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1334165632724762, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004826837219297886, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.36227947473526, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0019310874631628394, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4936541318893433, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8287.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2496962696313858, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8513751029968262, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8227750658988953, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7102102637290955, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11830786615610123, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.9547600746154785, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014067402109503746, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7168692946434021, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.032789308577775955, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7451581954956055, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9881.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8175002336502075, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.49649250507354736, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.379062652587891, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8002102375030518, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.13467827439308167, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.034112293273210526, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.508140563964844, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.08259426057338715, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.70645523071289, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029951414093375206, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09352898597717285, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09135918319225311, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01965664140880108, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14158105850219727, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018939394503831863, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9292879700660706, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20774301886558533, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0898633003234863, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1584.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7584403157234192, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.20611572265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9968.609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9968.4462890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26835936307907104, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 504257.3125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07265625149011612, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7937865257263184, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09626074880361557, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.639978408813477, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2560.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.576209545135498, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.5199545621871948, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.812280654907227, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.032957911491394, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03733590990304947, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03525754064321518, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.660747528076172, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.585685133934021, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.857858657836914, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013397719711065292, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09128317981958389, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08419054001569748, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012035857886075974, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13937892019748688, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005507710739038885, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11349654197692871, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005504827480763197, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5082395672798157, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19972.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14400534331798553, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6164719462394714, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.48808497190475464, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3752056360244751, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08697777986526489, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.848806381225586, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002888888819143176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24620933830738068, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0012717617210000753, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6273359060287476, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22500.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8735511302947998, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.27040529251098633, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.207940578460693, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.285976409912109, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.07037652283906937, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.0600525438785553, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.696104049682617, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.24145957827568054, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.273737907409668, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01567830704152584, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08974499255418777, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08346743881702423, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013696475885808468, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13786828517913818, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027142043691128492, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22588059306144714, "metrics/particle/momentum/all/accepted/pz_bias": -0.014174326322972775, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1397960186004639, "metrics/particle/momentum/all/accepted/samples": 29843.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21891120076179504, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7261015176773071, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 730.9107055664062, "metrics/particle/momentum/all/assigned/pt_relative_bias": 730.7942504882812, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10912681370973587, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 136491.15625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01116167288273573, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5659595131874084, "metrics/particle/momentum/all/assigned/pz_bias": 0.015506255440413952, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.05051851272583, "metrics/particle/momentum/all/assigned/samples": 34941.0, "metrics/particle/momentum/all/efficiency": 0.8347925543785095, "metrics/particle/momentum/all/event/energy_bias": -0.29386740922927856, "metrics/particle/momentum/all/event/p3_error_mean": 13.824857711791992, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.147512435913086, "metrics/particle/momentum/all/event/px_bias": -0.2423907220363617, "metrics/particle/momentum/all/event/py_bias": 0.05890733003616333, "metrics/particle/momentum/all/event/pz_abs_mean": 10.108423233032227, "metrics/particle/momentum/all/event/pz_bias": 1.4268198013305664, "metrics/particle/momentum/all/event/pz_residual_rms": 19.37594985961914, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008964254520833492, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04681634530425072, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.037289466708898544, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0038784644566476345, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08433791249990463, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015472212107852101, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14256222546100616, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.015906227752566338, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9248056411743164, "metrics/particle/momentum/class_1/accepted/samples": 16158.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07150114327669144, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21669608354568481, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2691039443016052, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21588721871376038, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04067223146557808, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.911063194274902, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005135087762027979, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2849905788898468, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012268411926925182, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.107701301574707, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9425421357154846, "metrics/particle/momentum/class_1/event/energy_bias": 0.6573610305786133, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.167271614074707, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.095429420471191, "metrics/particle/momentum/class_1/event/px_bias": 0.38010644912719727, "metrics/particle/momentum/class_1/event/py_bias": -0.053122490644454956, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.740390777587891, "metrics/particle/momentum/class_1/event/pz_bias": -0.0042837560176849365, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.435624122619629, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03554102033376694, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1969752460718155, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1988673061132431, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04203469306230545, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23935718834400177, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.035660091787576675, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6942813396453857, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13614311814308167, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.89847469329834, "metrics/particle/momentum/class_2/accepted/samples": 1318.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8278805017471313, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7045254707336426, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 9783.8984375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 9783.42578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36004599928855896, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 499595.34375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07285276055335999, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.961744546890259, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0020396600011736155, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.220141887664795, "metrics/particle/momentum/class_2/assigned/samples": 2608.0, "metrics/particle/momentum/class_2/efficiency": 0.4560553729534149, "metrics/particle/momentum/class_2/event/energy_bias": -6.192478179931641, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.627449035644531, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.876821041107178, "metrics/particle/momentum/class_2/event/px_bias": -0.11446744203567505, "metrics/particle/momentum/class_2/event/py_bias": 0.17256474494934082, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.042229652404785, "metrics/particle/momentum/class_2/event/pz_bias": -0.18955311179161072, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.04662036895752, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022795744240283966, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13529722392559052, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13369561731815338, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03261594846844673, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17609433829784393, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005875933566130698, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17132453620433807, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0006021028966642916, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5873653888702393, "metrics/particle/momentum/class_3/accepted/samples": 11913.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.28346115350723267, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2830381393432617, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.150619626045227, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0215171575546265, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14431795477867126, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.050612449645996, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0073624649085104465, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.47247517108917236, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01972622610628605, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7593038082122803, "metrics/particle/momentum/class_3/assigned/samples": 14669.0, "metrics/particle/momentum/class_3/efficiency": 0.7843176126480103, "metrics/particle/momentum/class_3/event/energy_bias": 6.396085262298584, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.389949798583984, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.062094688415527, "metrics/particle/momentum/class_3/event/px_bias": -0.3800121247768402, "metrics/particle/momentum/class_3/event/py_bias": -0.2454395294189453, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.693625450134277, "metrics/particle/momentum/class_3/event/pz_bias": 1.3148245811462402, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.650040626525879, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0115278959274292, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16244632005691528, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10542710870504379, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.044194404035806656, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15554608404636383, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007017544005066156, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.47451353073120117, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07583408057689667, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6228896379470825, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2933628261089325, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.802755355834961, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.392141342163086, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.325718402862549, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15819208323955536, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.186988830566406, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5711841583251953, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.134433314204216, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9201536178588867, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.805084764957428, "metrics/particle/momentum/class_4/event/energy_bias": -1.2312043905258179, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8613046407699585, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3772492408752441, "metrics/particle/momentum/class_4/event/px_bias": -0.16979597508907318, "metrics/particle/momentum/class_4/event/py_bias": 0.16299495100975037, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.013192057609558, "metrics/particle/momentum/class_4/event/pz_bias": 0.2650432884693146, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.821258068084717, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007982724346220493, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024230189621448517, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020862583070993423, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.001942143077030778, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05422123149037361, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16654440760612488, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0058864811435341835, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5909532904624939, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.015065597370266914, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03889046981930733, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03547527641057968, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0011684467317536473, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11369021981954575, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.197335347533226, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.00042600437882356346, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.6454429030418396, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": 0.07636824995279312, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7742245197296143, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6259623765945435, "metrics/particle/momentum/class_5/event/px_bias": 0.041778430342674255, "metrics/particle/momentum/class_5/event/py_bias": 0.021909618750214577, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.35054004192352295, "metrics/particle/momentum/class_5/event/pz_bias": 0.04078880697488785, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5480672121047974, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.606416940689087, "step_valid_time": 11581.442524909973, "step_total_time": 11583.04894208908, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998724225353415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07399869049146457, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2919, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2908, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9579914670167378, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9586206896551724, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9583059750492449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041379310344827586, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9543813587134887, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.955008210180624, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546946815495733, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9579914670167378, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013786208595978, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08182658827682987, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2169, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952610794207986, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528198376124644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04738920579201404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9521510096575944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9517332163229487, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9519420671494404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530289727831431, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063801502055003, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0774420970545695, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1228, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9317147192716236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.931008339651251, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.931361395525218, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06899166034874905, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9301972685887708, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9294920394238059, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9298445202882063, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9317147192716236} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_27000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..ad1e36bceaa73bc4783ce3149f06bf866fd78a2d --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2788935202728307, "train": {"Classification_binary": 2.4860455989837646, "Classification": 0.01770620420575142, "Regression_pt": 0.12749923765659332, "Regression_eta": 0.009478870779275894, "Regression_sin_phi": 0.006408622954040766, "Regression_cos_phi": 0.008074614219367504, "Regression_energy": 0.13055290281772614, "Total": 7.706333637237549, "Total_unweighted": 2.7857660516165197}, "train_group": {}, "valid": {"Classification_binary": 2.4851797421773276, "Classification": 0.018153253942728043, "Regression_pt": 0.13168028990427652, "Regression_eta": 0.01191058506568273, "Regression_sin_phi": 0.006671249866485596, "Regression_cos_phi": 0.0070497021079063416, "Regression_energy": 0.1368309756120046, "Total": 7.909121195475261, "Total_unweighted": 2.7974757986764116, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.018021846190094948, "diagnostic/loss/cld_pf/Classification_binary": 2.484786033630371, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006838759407401085, "diagnostic/loss/cld_pf/Regression_energy": 0.13528938591480255, "diagnostic/loss/cld_pf/Regression_eta": 0.010775291360914707, "diagnostic/loss/cld_pf/Regression_pt": 0.13069529831409454, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006257259286940098, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09488565474748611, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1925538182258606, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06776637583971024, "diagnostic/regression/cld_pf/energy_residual_rms": 0.42298609018325806, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.039957206696271896, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1870659738779068, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06712434440851212, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4147653877735138, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9416995644569397, "metrics/particle/class_1/pid_accuracy": 0.9861404895782471, "metrics/particle/class_2/efficiency": 0.4535718262195587, "metrics/particle/class_2/pid_accuracy": 0.7731725573539734, "metrics/particle/class_3/efficiency": 0.7814477682113647, "metrics/particle/class_3/pid_accuracy": 0.9846733808517456, "metrics/particle/class_4/efficiency": 0.8314833641052246, "metrics/particle/class_4/pid_accuracy": 0.39320388436317444, "metrics/particle/class_5/efficiency": 0.9786407947540283, "metrics/particle/class_5/pid_accuracy": 0.6547619104385376, "metrics/particle/count/bias_mean": -0.7032679915428162, "metrics/particle/count/mae": 3.215686321258545, "metrics/particle/count/prediction_mean": 69.86209106445312, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04936883598566055, "metrics/particle/event/energy_response_mean": 0.9882029294967651, "metrics/particle/event/met_abs_error": 4.80535364151001, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04004543647170067, "metrics/particle/event/scalar_pt_response_mean": 0.991856575012207, "metrics/particle/event/vector_pt_closure": 0.04411166161298752, "metrics/particle/matched/delta_eta_abs_mean": 0.00953260064125061, "metrics/particle/matched/delta_phi_abs_mean": 0.009754541330039501, "metrics/particle/matched/delta_r_mean": 0.01532546617090702, "metrics/particle/matched/energy_relative_abs_mean": 0.0892471969127655, "metrics/particle/matched/pid_accuracy": 0.968856930732727, "metrics/particle/matched/pt_relative_abs_mean": 0.08321145176887512, "metrics/particle/matching/duplicate_fraction": 0.03014341928064823, "metrics/particle/matching/efficiency": 0.8336405158042908, "metrics/particle/matching/f1": 0.8378154635429382, "metrics/particle/matching/purity": 0.8420323729515076, "metrics/particle/matching/target_coverage_dr0p05": 0.8183578252792358, "metrics/particle/matching/target_coverage_dr0p10": 0.8679757118225098, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7291622161865234, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8672069907188416, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017909852787852287, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08811819553375244, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08334702253341675, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017462454736232758, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13691924512386322, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0049367849715054035, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3792894780635834, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.00718908803537488, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5959351062774658, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8305.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2467157244682312, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7987475395202637, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.758546769618988, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6548138856887817, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11147373914718628, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.084990978240967, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01274859718978405, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6920125484466553, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.005292032845318317, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.535125255584717, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9805.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8192759156227112, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.2670743465423584, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.138659954071045, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.179826498031616, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.09596175700426102, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.04703865945339203, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.148223876953125, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.255055695772171, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.881451606750488, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026417767629027367, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09594117105007172, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09423620998859406, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01639307662844658, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14578117430210114, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0214198287576437, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9234474301338196, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18573448061943054, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9044077396392822, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1634.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7107871174812317, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.776946783065796, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17062.84375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17062.68359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25068870186805725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 859962.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07319952547550201, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.795722723007202, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.003841996192932129, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.662615776062012, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2541.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5943979620933533, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.9069647789001465, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.500800132751465, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0958330631256104, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01122882217168808, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08187191933393478, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.357499122619629, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.2133591175079346, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.564266204833984, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013313370756804943, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09163148701190948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08464722335338593, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01575501821935177, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13944722712039948, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006531678372994065, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11586828529834747, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.002225064905360341, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5385739803314209, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19903.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14121684432029724, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6129106283187866, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4861094653606415, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.37966349720954895, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08939866721630096, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.223949432373047, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002494432032108307, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23818339407444, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0009840200655162334, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0958850383758545, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22450.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8705331683158875, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.9860966205596924, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.651247978210449, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.521820068359375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4910534620285034, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.04700221121311188, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1753289699554443, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.34854361414909363, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.302558898925781, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015310097485780716, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09088971465826035, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08481042087078094, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016265133395791054, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13910433650016785, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029823738150298595, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23339715600013733, "metrics/particle/momentum/all/accepted/pz_bias": -0.013654613867402077, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1679797172546387, "metrics/particle/momentum/all/accepted/samples": 29842.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21253809332847595, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7502812147140503, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1246.552490234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1246.4427490234375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1073973998427391, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 232389.875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01054718904197216, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5528317093849182, "metrics/particle/momentum/all/assigned/pz_bias": 0.0018455316312611103, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0665884017944336, "metrics/particle/momentum/all/assigned/samples": 34796.0, "metrics/particle/momentum/all/efficiency": 0.8347645998001099, "metrics/particle/momentum/all/event/energy_bias": -3.1601357460021973, "metrics/particle/momentum/all/event/p3_error_mean": 14.135865211486816, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.965296268463135, "metrics/particle/momentum/all/event/px_bias": -0.4063205122947693, "metrics/particle/momentum/all/event/py_bias": 0.01216895878314972, "metrics/particle/momentum/all/event/pz_abs_mean": 10.700920104980469, "metrics/particle/momentum/all/event/pz_bias": 0.6097598075866699, "metrics/particle/momentum/all/event/pz_residual_rms": 21.110048294067383, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009315076284110546, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.048263389617204666, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.039304595440626144, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006267532240599394, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08772163093090057, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015477959532290697, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14178620278835297, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.009020073339343071, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8360671401023865, "metrics/particle/momentum/class_1/accepted/samples": 16152.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07467067241668701, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19359250366687775, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2124040573835373, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16017493605613708, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04078417643904686, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.9759204387664795, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004784409888088703, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2891499698162079, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.00390817504376173, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4761481285095215, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9421921372413635, "metrics/particle/momentum/class_1/event/energy_bias": 0.865973949432373, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.486213684082031, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.2154998779296875, "metrics/particle/momentum/class_1/event/px_bias": -0.07965898513793945, "metrics/particle/momentum/class_1/event/py_bias": -0.019254952669143677, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.994756698608398, "metrics/particle/momentum/class_1/event/pz_bias": -0.10913138091564178, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.110000610351562, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03420209884643555, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19909237325191498, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.200429305434227, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.055432386696338654, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2394331693649292, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04024297744035721, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8565298318862915, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1636163741350174, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.291714668273926, "metrics/particle/momentum/class_2/accepted/samples": 1317.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8149455785751343, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7273949980735779, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 16907.796875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 16907.341796875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3451637923717499, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 856096.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07215288281440735, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8741674423217773, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.10351037979125977, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.471158027648926, "metrics/particle/momentum/class_2/assigned/samples": 2564.0, "metrics/particle/momentum/class_2/efficiency": 0.4557093381881714, "metrics/particle/momentum/class_2/event/energy_bias": -7.982621669769287, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.777135848999023, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.8585357666015625, "metrics/particle/momentum/class_2/event/px_bias": 0.23420391976833344, "metrics/particle/momentum/class_2/event/py_bias": 0.0467105507850647, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.17204475402832, "metrics/particle/momentum/class_2/event/pz_bias": -0.5171157717704773, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.871990203857422, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021501827985048294, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1356954276561737, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1338396668434143, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.036630820482969284, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.175850972533226, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008392077870666981, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17451703548431396, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004054216202348471, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6224591135978699, "metrics/particle/momentum/class_3/accepted/samples": 11916.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2685739994049072, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3528695106506348, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2907949686050415, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1721665859222412, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14382809400558472, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.67722225189209, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006659343838691711, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4579271972179413, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.016356579959392548, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.792440414428711, "metrics/particle/momentum/class_3/assigned/samples": 14566.0, "metrics/particle/momentum/class_3/efficiency": 0.7845150828361511, "metrics/particle/momentum/class_3/event/energy_bias": 4.9584856033325195, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.85841178894043, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.698546409606934, "metrics/particle/momentum/class_3/event/px_bias": -0.47801950573921204, "metrics/particle/momentum/class_3/event/py_bias": -0.03693295270204544, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.420436859130859, "metrics/particle/momentum/class_3/event/pz_bias": 0.8850048780441284, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.258743286132812, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013538798317313194, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17182300984859467, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11691122502088547, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0689319372177124, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17290201783180237, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003448275849223137, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.44981035590171814, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0028749071061611176, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3354287147521973, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.29633718729019165, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.4179959297180176, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.819033622741699, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7701902389526367, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1525423675775528, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.533191680908203, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5698825120925903, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0084659019485116, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5997848510742188, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8192090392112732, "metrics/particle/momentum/class_4/event/energy_bias": -1.0701615810394287, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8036564588546753, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3168128728866577, "metrics/particle/momentum/class_4/event/px_bias": -0.15409694612026215, "metrics/particle/momentum/class_4/event/py_bias": 0.014659040607511997, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9996735453605652, "metrics/particle/momentum/class_4/event/pz_bias": 0.2581218481063843, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.117578506469727, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007428860291838646, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022757351398468018, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020129598677158356, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004025349859148264, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05450333654880524, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11896980553865433, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.01699262298643589, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.40728864073753357, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.053307656198740005, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04564297944307327, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04096904397010803, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.004661256447434425, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13787616789340973, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.22727911174297333, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.12363387644290924, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2687379121780396, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.0681878849864006, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8021965622901917, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6528046727180481, "metrics/particle/momentum/class_5/event/px_bias": 0.07125096023082733, "metrics/particle/momentum/class_5/event/py_bias": 0.0069872913882136345, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3580813407897949, "metrics/particle/momentum/class_5/event/pz_bias": 0.09288020431995392, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5910459756851196, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5976650714874268, "step_valid_time": 12029.553521871567, "step_total_time": 12031.15118765831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989769476334891, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07387429265966905, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2936, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9635707253035772, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.967699406723797, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9656306528531492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.032300593276203035, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596324253363965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.963744232036915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9616839335635586, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9635707253035772, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9963314719087258, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08117649654314485, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2172, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9577278731836195, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9562541217850077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04227212681638045, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534679543459175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9564068692206077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9549351505825456, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547848990342406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9979668229984991, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08047067076641956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9399239543726235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9388530193695405, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06007604562737642, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347496206373292, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9368821292775665, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9358146600835549, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377845220030349} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_28000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..0ca7df60f0d077e1a40d30111071ce34172ffc61 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3262599469496021, "train": {"Classification_binary": 2.4477741718292236, "Classification": 0.018356185406446457, "Regression_pt": 0.13855797052383423, "Regression_eta": 0.010046059265732765, "Regression_sin_phi": 0.006319283973425627, "Regression_cos_phi": 0.006532769184559584, "Regression_energy": 0.14345139265060425, "Total": 7.885855674743652, "Total_unweighted": 2.7710378328338265}, "train_group": {}, "valid": {"Classification_binary": 2.4839631716410318, "Classification": 0.01792181779940923, "Regression_pt": 0.1298364798227946, "Regression_eta": 0.01157096897562345, "Regression_sin_phi": 0.006597535063823064, "Regression_cos_phi": 0.00700073316693306, "Regression_energy": 0.1349120338757833, "Total": 7.829229990641276, "Total_unweighted": 2.7918027403453984, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017819903790950775, "diagnostic/loss/cld_pf/Classification_binary": 2.491774797439575, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006793893873691559, "diagnostic/loss/cld_pf/Regression_energy": 0.13367977738380432, "diagnostic/loss/cld_pf/Regression_eta": 0.010973528027534485, "diagnostic/loss/cld_pf/Regression_pt": 0.12991032004356384, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006123501341789961, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08759067952632904, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18793295323848724, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06047140434384346, "diagnostic/regression/cld_pf/energy_residual_rms": 0.417468786239624, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04544365406036377, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18584738671779633, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07261078804731369, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41347336769104004, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9452642798423767, "metrics/particle/class_1/pid_accuracy": 0.9843072295188904, "metrics/particle/class_2/efficiency": 0.4597242772579193, "metrics/particle/class_2/pid_accuracy": 0.7635687589645386, "metrics/particle/class_3/efficiency": 0.7864803075790405, "metrics/particle/class_3/pid_accuracy": 0.9853582382202148, "metrics/particle/class_4/efficiency": 0.8163471221923828, "metrics/particle/class_4/pid_accuracy": 0.42274412512779236, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.6892430186271667, "metrics/particle/count/bias_mean": 0.3052287697792053, "metrics/particle/count/mae": 3.1496732234954834, "metrics/particle/count/prediction_mean": 70.87059020996094, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0497472770512104, "metrics/particle/event/energy_response_mean": 1.0045044422149658, "metrics/particle/event/met_abs_error": 4.9118194580078125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04227191582322121, "metrics/particle/event/scalar_pt_response_mean": 1.0194151401519775, "metrics/particle/event/vector_pt_closure": 0.04362288489937782, "metrics/particle/matched/delta_eta_abs_mean": 0.010431205853819847, "metrics/particle/matched/delta_phi_abs_mean": 0.009611820802092552, "metrics/particle/matched/delta_r_mean": 0.01595199666917324, "metrics/particle/matched/energy_relative_abs_mean": 0.08586875349283218, "metrics/particle/matched/pid_accuracy": 0.968216598033905, "metrics/particle/matched/pt_relative_abs_mean": 0.08311981707811356, "metrics/particle/matching/duplicate_fraction": 0.03271174430847168, "metrics/particle/matching/efficiency": 0.8378270864486694, "metrics/particle/matching/f1": 0.836018979549408, "metrics/particle/matching/purity": 0.8342186808586121, "metrics/particle/matching/target_coverage_dr0p05": 0.8196267485618591, "metrics/particle/matching/target_coverage_dr0p10": 0.8740425109863281, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7328393459320068, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8715787529945374, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01901509240269661, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08305255323648453, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08287206292152405, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.027457065880298615, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13712629675865173, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005655836313962936, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3669998347759247, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00013221529661677778, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.555653691291809, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8310.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2503117024898529, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.084464430809021, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.15791916847229, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.063529133796692, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12088799476623535, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.031005859375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014328960329294205, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7065944075584412, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0411527045071125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5922186374664307, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9910.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8197691440582275, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.522129774093628, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.714853286743164, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9978067874908447, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15405036509037018, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.16493786871433258, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.801115989685059, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7629765272140503, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.975042343139648, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02654561772942543, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09103871136903763, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09229184687137604, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02814616449177265, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14333944022655487, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02216748706996441, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9641975164413452, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08175168931484222, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2411460876464844, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1624.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7428339719772339, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4110207557678223, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8436.263671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8436.11328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27042800188064575, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 427541.53125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07548638433218002, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.658971071243286, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04269411414861679, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.124197483062744, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2570.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5907602906227112, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.5270862579345703, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.204446792602539, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8883037567138672, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03188970685005188, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.040547534823417664, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.072633743286133, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.26788467168807983, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 12.678437232971191, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013471921905875206, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08825091272592545, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08411819487810135, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016682906076312065, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13889549672603607, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005488747847266495, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10777265578508377, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.012990498915314674, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4813114404678345, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20041.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13569876551628113, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6428406834602356, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5229610204696655, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4167221188545227, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09002788364887238, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.8464035987854, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002345859305933118, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2347709834575653, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.025699645280838013, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3598887920379639, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22593.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8765691518783569, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.9363596439361572, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.47592306137085, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.364560127258301, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.18223801255226135, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.2061372846364975, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.066286325454712, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0639559030532837, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.303571701049805, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015716973692178726, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0869608074426651, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08421555906534195, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020290900021791458, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13865254819393158, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031359465792775154, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22603826224803925, "metrics/particle/momentum/all/accepted/pz_bias": 0.004292795434594154, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1810791492462158, "metrics/particle/momentum/all/accepted/samples": 29975.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21257132291793823, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8239120244979858, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 618.8375854492188, "metrics/particle/momentum/all/assigned/pt_relative_bias": 618.7312622070312, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11196646839380264, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 115733.3203125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011091153137385845, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5457214117050171, "metrics/particle/momentum/all/assigned/pz_bias": 0.025054356083273888, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.609398365020752, "metrics/particle/momentum/all/assigned/samples": 35073.0, "metrics/particle/momentum/all/efficiency": 0.8384850025177002, "metrics/particle/momentum/all/event/energy_bias": 1.9314031600952148, "metrics/particle/momentum/all/event/p3_error_mean": 13.117868423461914, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.969928741455078, "metrics/particle/momentum/all/event/px_bias": 0.30439868569374084, "metrics/particle/momentum/all/event/py_bias": 0.08174693584442139, "metrics/particle/momentum/all/event/pz_abs_mean": 9.593818664550781, "metrics/particle/momentum/all/event/pz_bias": 2.0948171615600586, "metrics/particle/momentum/all/event/pz_residual_rms": 15.398252487182617, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009074296802282333, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04322896897792816, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03750930726528168, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008374691009521484, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08506932109594345, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016076175961643457, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13709934055805206, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.002572842640802264, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8565099835395813, "metrics/particle/momentum/class_1/accepted/samples": 16173.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0694865733385086, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24815236032009125, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.29485583305358887, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2485034465789795, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.040548425167798996, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.283229351043701, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0048424736596643925, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.26182615756988525, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005293209571391344, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.5639748573303223, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9434171319007874, "metrics/particle/momentum/class_1/event/energy_bias": 2.664108991622925, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.937469482421875, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.01329231262207, "metrics/particle/momentum/class_1/event/px_bias": 0.7142124772071838, "metrics/particle/momentum/class_1/event/py_bias": -0.010383069515228271, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.538991451263428, "metrics/particle/momentum/class_1/event/pz_bias": 0.11733195930719376, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.678786277770996, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034908611327409744, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1945393681526184, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19515974819660187, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03151926398277283, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23592501878738403, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04250559210777283, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7216989994049072, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11061657220125198, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.005972385406494, "metrics/particle/momentum/class_2/accepted/samples": 1341.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8284211754798889, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7305636405944824, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 8173.85498046875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 8173.39892578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37518852949142456, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 420879.8125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07390648871660233, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.873976707458496, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0491982027888298, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.596799373626709, "metrics/particle/momentum/class_2/assigned/samples": 2652.0, "metrics/particle/momentum/class_2/efficiency": 0.46401384472846985, "metrics/particle/momentum/class_2/event/energy_bias": -6.046599388122559, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.788122177124023, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.982501029968262, "metrics/particle/momentum/class_2/event/px_bias": -0.2247041016817093, "metrics/particle/momentum/class_2/event/py_bias": 0.21635258197784424, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.110286712646484, "metrics/particle/momentum/class_2/event/pz_bias": 0.3906821310520172, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.713951110839844, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022690385580062866, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13321833312511444, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13504621386528015, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.041375357657670975, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1770647168159485, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008325701346620917, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17564618587493896, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01644512079656124, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.813421368598938, "metrics/particle/momentum/class_3/accepted/samples": 12011.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26823675632476807, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4719792604446411, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.469208836555481, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3544995784759521, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14728209376335144, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.829404830932617, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00718449242413044, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.45840364694595337, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03758155554533005, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.317495107650757, "metrics/particle/momentum/class_3/assigned/samples": 14754.0, "metrics/particle/momentum/class_3/efficiency": 0.790769636631012, "metrics/particle/momentum/class_3/event/energy_bias": 6.196005821228027, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.757475852966309, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.857379913330078, "metrics/particle/momentum/class_3/event/px_bias": -0.24604371190071106, "metrics/particle/momentum/class_3/event/py_bias": -0.39202171564102173, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.178342819213867, "metrics/particle/momentum/class_3/event/pz_bias": 0.9316309690475464, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.006189346313477, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013320500031113625, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15205207467079163, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10719364136457443, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.059330351650714874, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1628710776567459, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0035587188322097063, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.419025719165802, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12213118374347687, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.264546275138855, "metrics/particle/momentum/class_4/accepted/samples": 281.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.301751047372818, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.774885892868042, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9386773109436035, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8794962167739868, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17231638729572296, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.03650951385498, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6843706369400024, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.27474433183670044, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.492565870285034, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7937853336334229, "metrics/particle/momentum/class_4/event/energy_bias": -1.156456470489502, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.213460922241211, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6335275173187256, "metrics/particle/momentum/class_4/event/px_bias": -0.10277535766363144, "metrics/particle/momentum/class_4/event/py_bias": 0.24537786841392517, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2113947868347168, "metrics/particle/momentum/class_4/event/pz_bias": 0.442585825920105, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.540175914764404, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007502704858779907, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022596662864089012, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02279789187014103, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008354417979717255, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05714605003595352, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12993143498897552, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.021074503660202026, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.43723106384277344, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.018245205283164978, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03692314401268959, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03561525419354439, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00416524987667799, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11122593283653259, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.1447770595550537, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.03349398076534271, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.4619099795818329, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": 0.27434423565864563, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8881410360336304, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6412014961242676, "metrics/particle/momentum/class_5/event/px_bias": 0.1637093722820282, "metrics/particle/momentum/class_5/event/py_bias": 0.022421253845095634, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4754457473754883, "metrics/particle/momentum/class_5/event/pz_bias": 0.21258626878261566, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.8842644691467285, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6469829082489014, "step_valid_time": 12474.731105804443, "step_total_time": 12476.37808895111, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0079086623338398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07427795143699178, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3038, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2923, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2912, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9593042336724648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9621461487820935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9607230895645029, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03785385121790652, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9556941253692156, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9585253456221198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9571076417419885, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9593042336724648, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0138915388889873, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08064343143036135, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2280, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2186, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2180, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9596136962247586, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9587719298245614, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591926283457657, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0412280701754386, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9569798068481123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956140350877193, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9565598946906538, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9596136962247586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.016273261303303, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08035721916884087, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0637329286798179, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936267071320182} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_29000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..83c7a39d1bb0b3a0134744598892e519cb3b99c4 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3736263736263736, "train": {"Classification_binary": 2.493577241897583, "Classification": 0.01841720938682556, "Regression_pt": 0.13201569020748138, "Regression_eta": 0.008092448115348816, "Regression_sin_phi": 0.006206749938428402, "Regression_cos_phi": 0.006682290695607662, "Regression_energy": 0.13453559577465057, "Total": 7.665718078613281, "Total_unweighted": 2.7995272260159254}, "train_group": {}, "valid": {"Classification_binary": 2.4656389554341636, "Classification": 0.017752387871344883, "Regression_pt": 0.12773374716440836, "Regression_eta": 0.011490825563669205, "Regression_sin_phi": 0.00657608484228452, "Regression_cos_phi": 0.006975993514060974, "Regression_energy": 0.1324401100476583, "Total": 7.743705749511719, "Total_unweighted": 2.7686081044375896, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01764226332306862, "diagnostic/loss/cld_pf/Classification_binary": 2.4633586406707764, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006713297683745623, "diagnostic/loss/cld_pf/Regression_energy": 0.1302938312292099, "diagnostic/loss/cld_pf/Regression_eta": 0.010536589659750462, "diagnostic/loss/cld_pf/Regression_pt": 0.12666985392570496, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006163853220641613, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0946558490395546, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18983374536037445, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06753657758235931, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4158186912536621, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.0437745600938797, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18522030115127563, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07094169408082962, "diagnostic/regression/cld_pf/pt_residual_rms": 0.41071105003356934, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9420445561408997, "metrics/particle/class_1/pid_accuracy": 0.986206591129303, "metrics/particle/class_2/efficiency": 0.45323002338409424, "metrics/particle/class_2/pid_accuracy": 0.7717446088790894, "metrics/particle/class_3/efficiency": 0.7830740213394165, "metrics/particle/class_3/pid_accuracy": 0.9854629039764404, "metrics/particle/class_4/efficiency": 0.8052472472190857, "metrics/particle/class_4/pid_accuracy": 0.3934836983680725, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6706587076187134, "metrics/particle/count/bias_mean": -0.48758170008659363, "metrics/particle/count/mae": 3.130718946456909, "metrics/particle/count/prediction_mean": 70.07777404785156, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05041461065411568, "metrics/particle/event/energy_response_mean": 0.9975501298904419, "metrics/particle/event/met_abs_error": 4.746109962463379, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03954638913273811, "metrics/particle/event/scalar_pt_response_mean": 1.0038377046585083, "metrics/particle/event/vector_pt_closure": 0.04311821237206459, "metrics/particle/matched/delta_eta_abs_mean": 0.008931374177336693, "metrics/particle/matched/delta_phi_abs_mean": 0.009222676046192646, "metrics/particle/matched/delta_r_mean": 0.014425369910895824, "metrics/particle/matched/energy_relative_abs_mean": 0.08805554360151291, "metrics/particle/matched/pid_accuracy": 0.9694328308105469, "metrics/particle/matched/pt_relative_abs_mean": 0.08260562270879745, "metrics/particle/matching/duplicate_fraction": 0.029948050156235695, "metrics/particle/matching/efficiency": 0.8341962695121765, "metrics/particle/matching/f1": 0.8370882868766785, "metrics/particle/matching/purity": 0.8400003910064697, "metrics/particle/matching/target_coverage_dr0p05": 0.8205066323280334, "metrics/particle/matching/target_coverage_dr0p10": 0.8693557977676392, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7320613265037537, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8683462142944336, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016734397038817406, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08468202501535416, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0812755674123764, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020232919603586197, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13382500410079956, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00506146065890789, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3616805672645569, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0034606517292559147, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5007473230361938, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8298.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24185925722122192, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8252164721488953, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8052086234092712, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7014360427856445, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11709696799516678, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.242382526397705, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014027241617441177, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.690180242061615, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.005311927758157253, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5205278396606445, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9838.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8185853958129883, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8588027358055115, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.347509860992432, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.773409128189087, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.06032189354300499, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.17423288524150848, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.516436576843262, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5189661383628845, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.107908248901367, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024105174466967583, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09144190698862076, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09085831046104431, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02134934440255165, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13988231122493744, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022560976445674896, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9094674587249756, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15491445362567902, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8524510860443115, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1640.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7205100655555725, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1986267566680908, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17145.271484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17145.11328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2597809135913849, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 866695.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07433489710092545, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5784595012664795, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2333388328552246, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.3437676429748535, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2556.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5965805649757385, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.771292805671692, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.619235992431641, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9617661833763123, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01936982199549675, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07610738277435303, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.466369152069092, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5381996035575867, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.74631118774414, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012451914139091969, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0906127393245697, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08364175260066986, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01811494491994381, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13797256350517273, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005531529895961285, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11001826822757721, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004234707448631525, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4979376494884491, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19886.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13775832951068878, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5872756838798523, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4357048571109772, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.33180734515190125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09022455662488937, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.2382612228393555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002851541619747877, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24380552768707275, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01816936396062374, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5370116233825684, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22444.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8697896003723145, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.12773042917251587, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.077317237854004, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.178647994995117, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.2196088582277298, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.07613968849182129, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.6089282035827637, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1922355592250824, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.346658229827881, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014284243807196617, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08900821954011917, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08338023722171783, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018882092088460922, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13693779706954956, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030177037697285414, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2239999771118164, "metrics/particle/momentum/all/accepted/pz_bias": -0.006657885853201151, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1132780313491821, "metrics/particle/momentum/all/accepted/samples": 29824.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20991110801696777, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6993221044540405, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1258.4249267578125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1258.3172607421875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11025317013263702, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 234758.03125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011252081021666527, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5411478281021118, "metrics/particle/momentum/all/assigned/pz_bias": -0.003914177417755127, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.696817398071289, "metrics/particle/momentum/all/assigned/samples": 34838.0, "metrics/particle/momentum/all/efficiency": 0.8342611193656921, "metrics/particle/momentum/all/event/energy_bias": -1.0402204990386963, "metrics/particle/momentum/all/event/p3_error_mean": 13.05971908569336, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.9174933433532715, "metrics/particle/momentum/all/event/px_bias": 0.13991713523864746, "metrics/particle/momentum/all/event/py_bias": -0.17426520586013794, "metrics/particle/momentum/all/event/pz_abs_mean": 9.337775230407715, "metrics/particle/momentum/all/event/pz_bias": 0.21146903932094574, "metrics/particle/momentum/all/event/pz_residual_rms": 17.519636154174805, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008482691831886768, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04774865135550499, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.038811974227428436, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009361072443425655, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.086322121322155, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013637490337714553, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13664422929286957, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.012730348855257034, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8030217289924622, "metrics/particle/momentum/class_1/accepted/samples": 16132.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07393744587898254, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.22765393555164337, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.23303534090518951, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18309694528579712, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04212368652224541, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.267167091369629, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0047841304913163185, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.27545273303985596, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.014356759376823902, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.065331220626831, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9410254955291748, "metrics/particle/momentum/class_1/event/energy_bias": 2.40885591506958, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.93351936340332, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.03147029876709, "metrics/particle/momentum/class_1/event/px_bias": 0.19237416982650757, "metrics/particle/momentum/class_1/event/py_bias": -0.06320935487747192, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.516504764556885, "metrics/particle/momentum/class_1/event/pz_bias": -0.4556660056114197, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.18172550201416, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0337480828166008, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19108401238918304, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19226416945457458, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05196637660264969, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23115591704845428, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.045836515724658966, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.765114188194275, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11970334500074387, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.0816264152526855, "metrics/particle/momentum/class_2/accepted/samples": 1309.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8210521340370178, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6778662204742432, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17010.8125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17010.35546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3711180090904236, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 863324.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07608695328235626, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.869764566421509, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.175882488489151, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.251008033752441, "metrics/particle/momentum/class_2/assigned/samples": 2576.0, "metrics/particle/momentum/class_2/efficiency": 0.4529411792755127, "metrics/particle/momentum/class_2/event/energy_bias": -7.300329208374023, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.604375839233398, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.819561958312988, "metrics/particle/momentum/class_2/event/px_bias": -0.09163770079612732, "metrics/particle/momentum/class_2/event/py_bias": 0.23223218321800232, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.007705688476562, "metrics/particle/momentum/class_2/event/pz_bias": -1.2362308502197266, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.522652626037598, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020145921036601067, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1325189620256424, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13177871704101562, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03845195844769478, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1737334132194519, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006704659899696708, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1706894487142563, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.013620579615235329, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6090796589851379, "metrics/particle/momentum/class_3/accepted/samples": 11932.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26292771100997925, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2134464979171753, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1338952779769897, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0171152353286743, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1436695009469986, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.385356903076172, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00760482344776392, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.44383129477500916, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.037696801126003265, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3066599369049072, "metrics/particle/momentum/class_3/assigned/samples": 14596.0, "metrics/particle/momentum/class_3/efficiency": 0.7855684757232666, "metrics/particle/momentum/class_3/event/energy_bias": 5.249222755432129, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.907275199890137, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.913951396942139, "metrics/particle/momentum/class_3/event/px_bias": 0.05416405200958252, "metrics/particle/momentum/class_3/event/py_bias": -0.54621422290802, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.278861045837402, "metrics/particle/momentum/class_3/event/pz_bias": 1.2892345190048218, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.237317085266113, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012261039577424526, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.172017440199852, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11570627242326736, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06929564476013184, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17166729271411896, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.37713396549224854, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.001599044888280332, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0085413455963135, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2520868480205536, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.819404125213623, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.717108130455017, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6658961772918701, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17280453443527222, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.392024040222168, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008498583920300007, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6340410113334656, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.04892362654209137, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2547433376312256, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.4631025791168213, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8660433292388916, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4673123359680176, "metrics/particle/momentum/class_4/event/px_bias": -0.08301209658384323, "metrics/particle/momentum/class_4/event/py_bias": 0.1912013292312622, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9199907779693604, "metrics/particle/momentum/class_4/event/pz_bias": 0.42383676767349243, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.4178411960601807, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006772219203412533, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024561898782849312, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02215215191245079, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009951652027666569, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.059453971683979034, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.131278857588768, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.008540540002286434, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4769308567047119, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.022461162880063057, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.046807773411273956, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.042503103613853455, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013826953247189522, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13148240745067596, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2124572992324829, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0272054560482502, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9176668524742126, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.06513333320617676, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.862206757068634, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6476770639419556, "metrics/particle/momentum/class_5/event/px_bias": 0.06802872568368912, "metrics/particle/momentum/class_5/event/py_bias": 0.011724906042218208, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4307332932949066, "metrics/particle/momentum/class_5/event/pz_bias": 0.19029471278190613, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7564234733581543, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.7001664638519287, "step_valid_time": 12921.202168226242, "step_total_time": 12922.902335166931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026579681563295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0719695797057337, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2917, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9644736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.96336454739609, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035526315789473684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9573350836888743, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9595394736842106, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9584360111713488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029658814242826, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07980842579797276, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582967515364355, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9595604395604396, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9589281792224906, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04043956043956044, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552238805970149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9564835164835165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9558532835493081, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582967515364355, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0080821090531833, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07759064688619599, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1240, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9408194233687405, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9429657794676806, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.941891378655526, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057034220532319393, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9406844106463879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9396126091910368, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9408194233687405} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_3000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..bc1f6519789a2b085b948c66c2ecde152c352f4b --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14209928003031452, "train": {"Classification_binary": 3.663644313812256, "Classification": 0.02836787886917591, "Regression_pt": 0.20602428913116455, "Regression_eta": 0.015171941369771957, "Regression_sin_phi": 0.009222181513905525, "Regression_cos_phi": 0.011786647140979767, "Regression_energy": 0.21141977608203888, "Total": 11.92529296875, "Total_unweighted": 4.1456370279192924}, "train_group": {}, "valid": {"Classification_binary": 3.687028249104818, "Classification": 0.02859414865573247, "Regression_pt": 0.20446276664733887, "Regression_eta": 0.014383145918448767, "Regression_sin_phi": 0.008411152909199396, "Regression_cos_phi": 0.008863485107819239, "Regression_energy": 0.2100352644920349, "Total": 11.629465738932291, "Total_unweighted": 4.161778212835391, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.028652120381593704, "diagnostic/loss/cld_pf/Classification_binary": 3.6760764122009277, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.008516348898410797, "diagnostic/loss/cld_pf/Regression_energy": 0.21203170716762543, "diagnostic/loss/cld_pf/Regression_eta": 0.013426079414784908, "diagnostic/loss/cld_pf/Regression_pt": 0.20767030119895935, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007911079563200474, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09109844267368317, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.23147642612457275, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06397915631532669, "diagnostic/regression/cld_pf/energy_residual_rms": 0.5002243518829346, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04819580540060997, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.22783321142196655, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07536294311285019, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4961665868759155, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9125685095787048, "metrics/particle/class_1/pid_accuracy": 0.9798597097396851, "metrics/particle/class_2/efficiency": 0.26364362239837646, "metrics/particle/class_2/pid_accuracy": 0.43863439559936523, "metrics/particle/class_3/efficiency": 0.7243099212646484, "metrics/particle/class_3/pid_accuracy": 0.9792469143867493, "metrics/particle/class_4/efficiency": 0.8758829236030579, "metrics/particle/class_4/pid_accuracy": 0.1463133692741394, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.4395161271095276, "metrics/particle/count/bias_mean": -1.7464052438735962, "metrics/particle/count/mae": 4.579084873199463, "metrics/particle/count/prediction_mean": 68.81895446777344, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.08104616403579712, "metrics/particle/event/energy_response_mean": 0.981167733669281, "metrics/particle/event/met_abs_error": 9.77834415435791, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06970177590847015, "metrics/particle/event/scalar_pt_response_mean": 0.9907474517822266, "metrics/particle/event/vector_pt_closure": 0.07765594869852066, "metrics/particle/matched/delta_eta_abs_mean": 0.012143122032284737, "metrics/particle/matched/delta_phi_abs_mean": 0.012839623726904392, "metrics/particle/matched/delta_r_mean": 0.019708044826984406, "metrics/particle/matched/energy_relative_abs_mean": 0.09413468837738037, "metrics/particle/matched/pid_accuracy": 0.9529868364334106, "metrics/particle/matched/pt_relative_abs_mean": 0.08940596878528595, "metrics/particle/matching/duplicate_fraction": 0.04497924819588661, "metrics/particle/matching/efficiency": 0.7803732752799988, "metrics/particle/matching/f1": 0.7901508808135986, "metrics/particle/matching/purity": 0.8001766204833984, "metrics/particle/matching/target_coverage_dr0p05": 0.7451118230819702, "metrics/particle/matching/target_coverage_dr0p10": 0.8160514831542969, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.676052451133728, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8197471499443054, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021186402067542076, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09018886834383011, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0874725952744484, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01567252352833748, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13998880982398987, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006199147552251816, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4122757911682129, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.031923312693834305, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8049585819244385, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7743.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3423323631286621, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3298594951629639, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.351230263710022, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.216347575187683, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1510821133852005, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.892618179321289, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02319560945034027, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0038154125213623, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.079442098736763, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.457099437713623, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9657.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7638354301452637, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9115713238716125, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.393278121948242, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.583920478820801, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.07596038281917572, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.8039810061454773, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.951662063598633, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.207166910171509, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.291115760803223, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03004639409482479, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1060931608080864, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10392707586288452, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.029995979741215706, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15538762509822845, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023019634187221527, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9870603680610657, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12822461128234863, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2737855911254883, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1477.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8522700667381287, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.0234529972076416, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 13979.951171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 13979.7841796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.30263692140579224, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 693832.5625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08884381502866745, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2964296340942383, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07105700671672821, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.123113632202148, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2465.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5372862815856934, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.27769750356674194, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.528970718383789, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4887361526489258, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.021584054455161095, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.020781509578227997, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.327369689941406, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.119942307472229, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.890844345092773, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01857474446296692, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09731952100992203, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09185666590929031, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013915305025875568, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14734406769275665, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006392158684320748, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.129190593957901, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.018366267904639244, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5824731588363647, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18773.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20705977082252502, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0989491939544678, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7873268127441406, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6503768563270569, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11991676688194275, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.814760208129883, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006061428692191839, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4023025929927826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.010242522694170475, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.277775526046753, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22107.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.821108341217041, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.0423076152801514, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.573390007019043, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.598088264465332, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.6426613926887512, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.9389753341674805, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.654842853546143, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8851701021194458, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.708527565002441, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01990242302417755, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09581007063388824, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09128088504076004, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01524982787668705, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14578811824321747, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003357982262969017, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2527572810649872, "metrics/particle/momentum/all/accepted/pz_bias": -0.027912687510252, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3016033172607422, "metrics/particle/momentum/all/accepted/samples": 27993.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2916889190673828, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2306736707687378, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1007.6553955078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1007.5169067382812, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1418680101633072, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 186194.15625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016857050359249115, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7804272770881653, "metrics/particle/momentum/all/assigned/pz_bias": -0.0341452993452549, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.0379109382629395, "metrics/particle/momentum/all/assigned/samples": 34229.0, "metrics/particle/momentum/all/efficiency": 0.7830429077148438, "metrics/particle/momentum/all/event/energy_bias": -3.231576442718506, "metrics/particle/momentum/all/event/p3_error_mean": 23.484817504882812, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.343086242675781, "metrics/particle/momentum/all/event/px_bias": -0.7402058839797974, "metrics/particle/momentum/all/event/py_bias": 3.763737678527832, "metrics/particle/momentum/all/event/pz_abs_mean": 16.259305953979492, "metrics/particle/momentum/all/event/pz_bias": -2.9723947048187256, "metrics/particle/momentum/all/event/pz_residual_rms": 25.873456954956055, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012147570960223675, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.057330016046762466, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.049369774758815765, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0129855303093791, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10035225003957748, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018568318337202072, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16734889149665833, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.004278765060007572, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9736636281013489, "metrics/particle/momentum/class_1/accepted/samples": 15618.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11945455521345139, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3303440511226654, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3874049782752991, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3146073818206787, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0675652027130127, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.370073318481445, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008051811717450619, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.42269453406333923, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02289835549890995, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6248435974121094, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9110423922538757, "metrics/particle/momentum/class_1/event/energy_bias": 2.8202836513519287, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.255163192749023, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.027475357055664, "metrics/particle/momentum/class_1/event/px_bias": -0.5254561901092529, "metrics/particle/momentum/class_1/event/py_bias": 0.08506402373313904, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.647581100463867, "metrics/particle/momentum/class_1/event/pz_bias": -0.562370240688324, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.64069938659668, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04467898979783058, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22409294545650482, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2228245586156845, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06118733435869217, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2612161636352539, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.06377550959587097, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.5868146419525146, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.4730185568332672, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.659718990325928, "metrics/particle/momentum/class_2/accepted/samples": 784.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1697636842727661, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.03628408908844, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 15156.853515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 15156.251953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4901011884212494, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 722542.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.10822701454162598, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.174620151519775, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.23631015419960022, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.648272514343262, "metrics/particle/momentum/class_2/assigned/samples": 2273.0, "metrics/particle/momentum/class_2/efficiency": 0.27128028869628906, "metrics/particle/momentum/class_2/event/energy_bias": -12.769404411315918, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.297697067260742, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.99560546875, "metrics/particle/momentum/class_2/event/px_bias": -0.3701263964176178, "metrics/particle/momentum/class_2/event/py_bias": 2.3845131397247314, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.307300567626953, "metrics/particle/momentum/class_2/event/pz_bias": -2.1261303424835205, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.793947219848633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029291978105902672, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13996970653533936, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1412438601255417, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023144569247961044, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18362779915332794, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011694853892549872, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1986335664987564, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03101443313062191, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7304474711418152, "metrics/particle/momentum/class_3/accepted/samples": 11116.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3647376298904419, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.3338217735290527, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2535128593444824, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.107067584991455, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1781540811061859, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 26.4563045501709, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013085158541798592, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6718193888664246, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.017577165737748146, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.303195476531982, "metrics/particle/momentum/class_3/assigned/samples": 14291.0, "metrics/particle/momentum/class_3/efficiency": 0.7318454384803772, "metrics/particle/momentum/class_3/event/energy_bias": 8.990859031677246, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.522087097167969, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.154776573181152, "metrics/particle/momentum/class_3/event/px_bias": -0.07911641895771027, "metrics/particle/momentum/class_3/event/py_bias": 0.9815144538879395, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.686518669128418, "metrics/particle/momentum/class_3/event/pz_bias": -0.7016039490699768, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.45635986328125, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017049318179488182, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16393671929836273, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11439651995897293, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.043805625289678574, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16855888068675995, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006493506487458944, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6146922707557678, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.00022167354472912848, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2106070518493652, "metrics/particle/momentum/class_4/accepted/samples": 308.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1479542851448059, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.116164445877075, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8667691946029663, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7865370512008667, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.09631728380918503, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.910154342651367, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016997167840600014, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9100713729858398, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.06052669882774353, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.8288419246673584, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8700565099716187, "metrics/particle/momentum/class_4/event/energy_bias": -2.4151406288146973, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.3882927894592285, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.4488601684570312, "metrics/particle/momentum/class_4/event/px_bias": 0.09078628569841385, "metrics/particle/momentum/class_4/event/py_bias": 0.24637041985988617, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.940392255783081, "metrics/particle/momentum/class_4/event/pz_bias": 0.29478543996810913, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.788095474243164, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009093624539673328, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.027225369587540627, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024989627301692963, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007689501158893108, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06080099940299988, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.21784985065460205, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.005991543177515268, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6438121199607849, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07700242847204208, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04529638588428497, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04360546916723251, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.006049572490155697, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12956343591213226, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.3326454758644104, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0540008582174778, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.0698072910308838, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.1418267786502838, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6640746593475342, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1862246990203857, "metrics/particle/momentum/class_5/event/px_bias": 0.14370688796043396, "metrics/particle/momentum/class_5/event/py_bias": 0.0662757083773613, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9799343943595886, "metrics/particle/momentum/class_5/event/pz_bias": 0.12292467057704926, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.08317232131958, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5350141525268555, "step_valid_time": 1337.8922073841095, "step_total_time": 1339.427221775055, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9929032800025265, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15640127965511852, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2803, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2770, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9199212340006564, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9281456953642384, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9240151640019779, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07185430463576159, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9090909090909091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9172185430463576, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9131366408439097, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9199212340006564, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9907797698136183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15713767636944498, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2090, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9174714661984197, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9301290609701824, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9237569060773481, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06987093902981753, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9069359086918349, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9194481530930129, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9131491712707183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9174714661984197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995474428579685, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15218291957737962, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1300, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1164, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8915022761760243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9038461538461539, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8976317799847212, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09615384615384616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8831562974203339, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8953846153846153, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8892284186401833, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8915022761760243} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_30000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..89c13e11cf6f2ba2015e7fb85edc9cea47c77d38 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4209928003031451, "train": {"Classification_binary": 2.473389148712158, "Classification": 0.017582865431904793, "Regression_pt": 0.12664033472537994, "Regression_eta": 0.012689761817455292, "Regression_sin_phi": 0.008940443396568298, "Regression_cos_phi": 0.007063169497996569, "Regression_energy": 0.13337314128875732, "Total": 7.95515775680542, "Total_unweighted": 2.7796788648702204}, "train_group": {}, "valid": {"Classification_binary": 2.4629222551981607, "Classification": 0.017807879795630772, "Regression_pt": 0.12799358367919922, "Regression_eta": 0.011805848528941473, "Regression_sin_phi": 0.006544935206572215, "Regression_cos_phi": 0.0068824390570322675, "Regression_energy": 0.13315985600153604, "Total": 7.761499404907227, "Total_unweighted": 2.7671167974670725, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017736472189426422, "diagnostic/loss/cld_pf/Classification_binary": 2.464092493057251, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006571216508746147, "diagnostic/loss/cld_pf/Regression_energy": 0.1319224238395691, "diagnostic/loss/cld_pf/Regression_eta": 0.01060212031006813, "diagnostic/loss/cld_pf/Regression_pt": 0.12771494686603546, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006162991281598806, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09729773551225662, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.19086308777332306, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07017846405506134, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4214748442173004, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04485536739230156, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18568968772888184, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07202250510454178, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4134073257446289, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9452834725379944, "metrics/particle/class_1/pid_accuracy": 0.9857267141342163, "metrics/particle/class_2/efficiency": 0.45983821153640747, "metrics/particle/class_2/pid_accuracy": 0.7792368531227112, "metrics/particle/class_3/efficiency": 0.7753604054450989, "metrics/particle/class_3/pid_accuracy": 0.9847230911254883, "metrics/particle/class_4/efficiency": 0.8213925361633301, "metrics/particle/class_4/pid_accuracy": 0.41646191477775574, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6739562749862671, "metrics/particle/count/bias_mean": -0.6169934868812561, "metrics/particle/count/mae": 3.2117648124694824, "metrics/particle/count/prediction_mean": 69.9483642578125, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04872185364365578, "metrics/particle/event/energy_response_mean": 0.9934996366500854, "metrics/particle/event/met_abs_error": 4.940256595611572, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03807459771633148, "metrics/particle/event/scalar_pt_response_mean": 1.0001651048660278, "metrics/particle/event/vector_pt_closure": 0.042902737855911255, "metrics/particle/matched/delta_eta_abs_mean": 0.008972711861133575, "metrics/particle/matched/delta_phi_abs_mean": 0.009083615615963936, "metrics/particle/matched/delta_r_mean": 0.014366235584020615, "metrics/particle/matched/energy_relative_abs_mean": 0.08780861645936966, "metrics/particle/matched/pid_accuracy": 0.9691745042800903, "metrics/particle/matched/pt_relative_abs_mean": 0.08241681754589081, "metrics/particle/matching/duplicate_fraction": 0.02983526512980461, "metrics/particle/matching/efficiency": 0.8332144618034363, "metrics/particle/matching/f1": 0.8368731141090393, "metrics/particle/matching/purity": 0.8405640125274658, "metrics/particle/matching/target_coverage_dr0p05": 0.8189505934715271, "metrics/particle/matching/target_coverage_dr0p10": 0.8677719831466675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7318019866943359, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.868198037147522, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01653260551393032, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0843183696269989, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08045938611030579, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019930830225348473, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13354380428791046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004949897527694702, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3659174144268036, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.004335972014814615, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5399596691131592, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8283.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24621336162090302, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8811905384063721, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7993564605712891, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6966519355773926, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11258548498153687, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.017313003540039, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01408594474196434, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6954157948493958, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.026126369833946228, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5370447635650635, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9797.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8171056509017944, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7435689568519592, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.643277645111084, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9122800827026367, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.08324863761663437, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.10239426791667938, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.778609752655029, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4995468258857727, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.043240547180176, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02583376131951809, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09419232606887817, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09357637912034988, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018620114773511887, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14619462192058563, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025061124935746193, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.023537516593933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.11041349172592163, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.261946439743042, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1636.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7317532896995544, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5173438787460327, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 16904.31640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 16904.15234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25375494360923767, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 850127.3125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07588932663202286, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7252137660980225, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.06644667685031891, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.876852512359619, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2530.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5951254963874817, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.7467007637023926, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.797024726867676, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9645166397094727, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04301830753684044, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.009978631511330605, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.668177127838135, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.3743569850921631, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.327547073364258, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012428329326212406, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09013879299163818, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08371584862470627, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019096197560429573, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13871438801288605, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005029928288422525, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1123335212469101, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00202724477276206, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5045965313911438, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19881.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1394517421722412, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5963318347930908, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4495961666107178, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.34748128056526184, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0884857103228569, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.7161054611206055, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002407168038189411, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23392580449581146, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.005750416312366724, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.065627932548523, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22433.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8695709109306335, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.5580049157142639, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.236813068389893, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.270353317260742, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5296702980995178, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.11294417083263397, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.8267323970794678, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4384716749191284, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.742804050445557, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014305075630545616, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08874352276325226, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08335203677415848, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019302047789096832, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1377221792936325, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003087248420342803, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23284240067005157, "metrics/particle/momentum/all/accepted/pz_bias": -0.0035039582289755344, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1887688636779785, "metrics/particle/momentum/all/accepted/samples": 29800.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21265272796154022, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7436541318893433, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1230.8929443359375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1230.7861328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10730724781751633, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 229352.96875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011047180742025375, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5453232526779175, "metrics/particle/momentum/all/assigned/pz_bias": -0.0062384651973843575, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.657613754272461, "metrics/particle/momentum/all/assigned/samples": 34760.0, "metrics/particle/momentum/all/efficiency": 0.8335897326469421, "metrics/particle/momentum/all/event/energy_bias": -1.5611367225646973, "metrics/particle/momentum/all/event/p3_error_mean": 13.69752025604248, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.974879264831543, "metrics/particle/momentum/all/event/px_bias": 0.4894399642944336, "metrics/particle/momentum/all/event/py_bias": 0.0005712807178497314, "metrics/particle/momentum/all/event/pz_abs_mean": 10.199230194091797, "metrics/particle/momentum/all/event/pz_bias": -1.312375545501709, "metrics/particle/momentum/all/event/pz_residual_rms": 17.473346710205078, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008310976438224316, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0459512360394001, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03754168003797531, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0058449637144804, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08506506681442261, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016681082779541612, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14088647067546844, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0017594019882380962, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8994460105895996, "metrics/particle/momentum/class_1/accepted/samples": 16186.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07304640114307404, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20998670160770416, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.23991207778453827, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18707963824272156, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.038685962557792664, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.334923267364502, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0047846888191998005, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.26447853446006775, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.013124040327966213, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7143690586090088, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9441754817962646, "metrics/particle/momentum/class_1/event/energy_bias": 0.6117041110992432, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.165574073791504, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.200588703155518, "metrics/particle/momentum/class_1/event/px_bias": 0.4512978196144104, "metrics/particle/momentum/class_1/event/py_bias": 0.02308669686317444, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.584958076477051, "metrics/particle/momentum/class_1/event/pz_bias": -0.5478009581565857, "metrics/particle/momentum/class_1/event/pz_residual_rms": 12.19625186920166, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03436547517776489, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1904057413339615, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19168606400489807, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04471578076481819, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2322651743888855, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03978978842496872, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7833017110824585, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1374637633562088, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.175848007202148, "metrics/particle/momentum/class_2/accepted/samples": 1332.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8269389867782593, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7700470685958862, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 16613.71484375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 16613.26953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35353535413742065, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 842829.9375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07342657446861267, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9120426177978516, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.075833760201931, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.46573543548584, "metrics/particle/momentum/class_2/assigned/samples": 2574.0, "metrics/particle/momentum/class_2/efficiency": 0.4608996510505676, "metrics/particle/momentum/class_2/event/energy_bias": -6.2751922607421875, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.158987045288086, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.0785698890686035, "metrics/particle/momentum/class_2/event/px_bias": 0.2941893935203552, "metrics/particle/momentum/class_2/event/py_bias": 0.37571436166763306, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.482586860656738, "metrics/particle/momentum/class_2/event/pz_bias": -2.0237629413604736, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.402993202209473, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020442655310034752, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1352056860923767, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13422591984272003, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04454242065548897, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17655043303966522, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007608419982716441, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1790570169687271, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.004737849347293377, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6488964557647705, "metrics/particle/momentum/class_3/accepted/samples": 11829.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2699837386608124, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3408399820327759, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2035781145095825, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0911715030670166, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1444704532623291, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.451138496398926, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007368131075054407, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.45916664600372314, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.012777947820723057, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.432950258255005, "metrics/particle/momentum/class_3/assigned/samples": 14522.0, "metrics/particle/momentum/class_3/efficiency": 0.7787872552871704, "metrics/particle/momentum/class_3/event/energy_bias": 4.94246244430542, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.888765335083008, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.941218376159668, "metrics/particle/momentum/class_3/event/px_bias": -0.28968560695648193, "metrics/particle/momentum/class_3/event/py_bias": -0.6393662691116333, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.263972282409668, "metrics/particle/momentum/class_3/event/pz_bias": 0.786620557308197, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.989683151245117, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011065805330872536, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1528633087873459, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10273199528455734, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04842803627252579, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15511558949947357, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010489510372281075, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4745287001132965, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1469540297985077, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5912646055221558, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2514702379703522, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2343993186950684, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5235120058059692, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4672940969467163, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15297450125217438, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.23880672454834, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016997167840600014, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6369670629501343, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.03427986428141594, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0446157455444336, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.0703604221343994, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.948932409286499, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.419865369796753, "metrics/particle/momentum/class_4/event/px_bias": -0.09161923825740814, "metrics/particle/momentum/class_4/event/py_bias": 0.18469835817813873, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0862255096435547, "metrics/particle/momentum/class_4/event/pz_bias": 0.2569102346897125, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.195411205291748, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006071743089705706, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024567866697907448, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02260763570666313, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0035138411913067102, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06267223507165909, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17471259832382202, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.05442264676094055, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7175297737121582, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.011130236089229584, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0470719076693058, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04323582723736763, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0018795799696817994, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13485972583293915, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.19849413633346558, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.03239715099334717, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.7378324866294861, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.23024961352348328, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9056315422058105, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6532455682754517, "metrics/particle/momentum/class_5/event/px_bias": 0.12525762617588043, "metrics/particle/momentum/class_5/event/py_bias": 0.05643817037343979, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4719151258468628, "metrics/particle/momentum/class_5/event/pz_bias": 0.21565748751163483, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.8617231845855713, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5893681049346924, "step_valid_time": 13367.366173505783, "step_total_time": 13368.955541849136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019440610604884, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0743784645483232, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2941, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.965211683623236, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9667981591058514, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.966004269995073, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033201840894148585, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9619297669839186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9635108481262328, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9627196584003942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.965211683623236, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019813016427315, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0846104729149838, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2281, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2180, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2172, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9569798068481123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9557211749232792, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9563500767712217, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044278825076720735, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534679543459175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9522139412538361, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9528405352050888, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9569798068481123, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0075181059550822, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07901950558179549, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1240, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9408194233687405, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9372637944066515, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9390382430897387, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06273620559334846, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9357520786092215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9375236652783037, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9408194233687405} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_31000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..3d6da043f5e83fc46221e5164331dfbd9a3a1850 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4683592269799166, "train": {"Classification_binary": 2.4650027751922607, "Classification": 0.018963122740387917, "Regression_pt": 0.1346047818660736, "Regression_eta": 0.010321422480046749, "Regression_sin_phi": 0.007006595842540264, "Regression_cos_phi": 0.006886107847094536, "Regression_energy": 0.13374820351600647, "Total": 7.8593549728393555, "Total_unweighted": 2.7765330094844103}, "train_group": {}, "valid": {"Classification_binary": 2.455766201019287, "Classification": 0.017746274669965107, "Regression_pt": 0.1260426938533783, "Regression_eta": 0.011180656651655832, "Regression_sin_phi": 0.006493045638004939, "Regression_cos_phi": 0.006864967445532481, "Regression_energy": 0.1310454805692037, "Total": 7.669679005940755, "Total_unweighted": 2.7551393198470273, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017661068588495255, "diagnostic/loss/cld_pf/Classification_binary": 2.45849871635437, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006639670580625534, "diagnostic/loss/cld_pf/Regression_energy": 0.1294160634279251, "diagnostic/loss/cld_pf/Regression_eta": 0.010312702506780624, "diagnostic/loss/cld_pf/Regression_pt": 0.1251925230026245, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00619844114407897, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09376150369644165, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18813739717006683, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06664222478866577, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4157964885234833, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03924459591507912, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18271423876285553, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06641173362731934, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40711429715156555, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9436544179916382, "metrics/particle/class_1/pid_accuracy": 0.9863520264625549, "metrics/particle/class_2/efficiency": 0.4556226432323456, "metrics/particle/class_2/pid_accuracy": 0.7606901526451111, "metrics/particle/class_3/efficiency": 0.7829641103744507, "metrics/particle/class_3/pid_accuracy": 0.9860222339630127, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.4070904552936554, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6906187534332275, "metrics/particle/count/bias_mean": -1.0111111402511597, "metrics/particle/count/mae": 3.3052287101745605, "metrics/particle/count/prediction_mean": 69.55424499511719, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04951181262731552, "metrics/particle/event/energy_response_mean": 0.9963697791099548, "metrics/particle/event/met_abs_error": 4.902256488800049, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039510056376457214, "metrics/particle/event/scalar_pt_response_mean": 1.002160668373108, "metrics/particle/event/vector_pt_closure": 0.04324943572282791, "metrics/particle/matched/delta_eta_abs_mean": 0.009240743704140186, "metrics/particle/matched/delta_phi_abs_mean": 0.008867123164236546, "metrics/particle/matched/delta_r_mean": 0.014446457847952843, "metrics/particle/matched/energy_relative_abs_mean": 0.08853071928024292, "metrics/particle/matched/pid_accuracy": 0.9693182706832886, "metrics/particle/matched/pt_relative_abs_mean": 0.08275599032640457, "metrics/particle/matching/duplicate_fraction": 0.02869815193116665, "metrics/particle/matching/efficiency": 0.8353077173233032, "metrics/particle/matching/f1": 0.8413353562355042, "metrics/particle/matching/purity": 0.8474506139755249, "metrics/particle/matching/target_coverage_dr0p05": 0.8197100758552551, "metrics/particle/matching/target_coverage_dr0p10": 0.8674570322036743, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7331820726394653, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8697170615196228, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016523517668247223, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08462050557136536, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07984933257102966, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018245356157422066, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13149452209472656, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004728419240564108, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3616750240325928, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.00585862435400486, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4732221364974976, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8248.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2502897083759308, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9572891592979431, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0185469388961792, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9134011268615723, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11229726672172546, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.192728996276855, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012831040658056736, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6856906414031982, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.01878560520708561, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4806175231933594, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9742.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8136529326438904, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.05077233165502548, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.3609619140625, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1947288513183594, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.06729752570390701, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2089167833328247, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.407621383666992, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3221331238746643, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.587608337402344, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025079892948269844, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09252789616584778, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09028899669647217, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01665748842060566, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14144639670848846, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023284314200282097, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9532714486122131, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08053996413946152, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2631282806396484, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1632.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6998019218444824, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7758022546768188, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5428.154296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5427.99462890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24379503726959229, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 271164.46875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07686148583889008, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.744267702102661, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.057225096970796585, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.568989753723145, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2498.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5936704277992249, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7461307048797607, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.052543640136719, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0143271684646606, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07396391034126282, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.012196104973554611, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.912790298461914, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3339899182319641, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.657325744628906, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012597085908055305, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0904984176158905, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08384077996015549, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018828360363841057, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13809430599212646, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00030096308910287917, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10906411707401276, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.006975418888032436, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.46437695622444153, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19936.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13471685349941254, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6054558753967285, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4470757842063904, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3443969488143921, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08536476641893387, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.507158279418945, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0022323422599583864, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2345002442598343, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013208522461354733, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2092195749282837, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22398.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8719765543937683, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.3419801890850067, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.264613628387451, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.173697471618652, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.12594108283519745, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.40494850277900696, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.008570909500122, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.01083393394947052, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.801201343536377, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014366510324180126, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08898349106311798, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08308958262205124, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018548259511590004, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13648860156536102, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002783740172162652, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2251521497964859, "metrics/particle/momentum/all/accepted/pz_bias": -0.0013650789624080062, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1521317958831787, "metrics/particle/momentum/all/accepted/samples": 29816.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20797434449195862, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7888119220733643, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 392.0395812988281, "metrics/particle/momentum/all/assigned/pt_relative_bias": 391.93206787109375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10436514765024185, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 72820.3203125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010595300234854221, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.542396068572998, "metrics/particle/momentum/all/assigned/pz_bias": -0.0008693680283613503, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.823399543762207, "metrics/particle/momentum/all/assigned/samples": 34638.0, "metrics/particle/momentum/all/efficiency": 0.8340373039245605, "metrics/particle/momentum/all/event/energy_bias": -0.45492297410964966, "metrics/particle/momentum/all/event/p3_error_mean": 13.227365493774414, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.755370616912842, "metrics/particle/momentum/all/event/px_bias": 0.13260746002197266, "metrics/particle/momentum/all/event/py_bias": -0.6260614395141602, "metrics/particle/momentum/all/event/pz_abs_mean": 9.837017059326172, "metrics/particle/momentum/all/event/pz_bias": 0.022690683603286743, "metrics/particle/momentum/all/event/pz_residual_rms": 17.95560073852539, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008153034374117851, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04683065414428711, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03808818757534027, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00971687026321888, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08495035022497177, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001176470541395247, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1373564600944519, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0022635182831436396, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8293773531913757, "metrics/particle/momentum/class_1/accepted/samples": 16150.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0760294571518898, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19226211309432983, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21900689601898193, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17016975581645966, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04120943322777748, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.418950080871582, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0037940696347504854, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.27098095417022705, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.010390475392341614, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6795767545700073, "metrics/particle/momentum/class_1/assigned/samples": 17132.0, "metrics/particle/momentum/class_1/efficiency": 0.9420754909515381, "metrics/particle/momentum/class_1/event/energy_bias": 3.231248617172241, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.999094009399414, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.072965621948242, "metrics/particle/momentum/class_1/event/px_bias": 0.5823843479156494, "metrics/particle/momentum/class_1/event/py_bias": -0.47791463136672974, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.578711986541748, "metrics/particle/momentum/class_1/event/pz_bias": -0.6511192321777344, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.477309226989746, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03446152061223984, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1919003129005432, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19292183220386505, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04585355892777443, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2327260971069336, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.042585551738739014, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.711238980293274, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.15123365819454193, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.011312007904053, "metrics/particle/momentum/class_2/accepted/samples": 1315.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7917205691337585, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7181143760681152, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5413.47265625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5413.025390625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3414536714553833, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 270839.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07827475666999817, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8957252502441406, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09737447649240494, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.64557409286499, "metrics/particle/momentum/class_2/assigned/samples": 2504.0, "metrics/particle/momentum/class_2/efficiency": 0.4550172984600067, "metrics/particle/momentum/class_2/event/energy_bias": -8.489384651184082, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.706170082092285, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.974900245666504, "metrics/particle/momentum/class_2/event/px_bias": 0.07636301219463348, "metrics/particle/momentum/class_2/event/py_bias": -0.04873821139335632, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.987824440002441, "metrics/particle/momentum/class_2/event/pz_bias": -1.4128884077072144, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.81205940246582, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020715441554784775, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1338038146495819, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13223043084144592, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03667913377285004, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17381687462329865, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005884330603294075, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17678652703762054, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.009136284701526165, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7595653533935547, "metrics/particle/momentum/class_3/accepted/samples": 11896.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26411277055740356, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4756710529327393, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3702644109725952, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2504749298095703, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13754749298095703, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.037324905395508, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007115716580301523, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.46019959449768066, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.023718059062957764, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.8322930335998535, "metrics/particle/momentum/class_3/assigned/samples": 14475.0, "metrics/particle/momentum/class_3/efficiency": 0.783198356628418, "metrics/particle/momentum/class_3/event/energy_bias": 5.9478759765625, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.003244400024414, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.811952114105225, "metrics/particle/momentum/class_3/event/px_bias": -0.4466233253479004, "metrics/particle/momentum/class_3/event/py_bias": -0.19661018252372742, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.457466125488281, "metrics/particle/momentum/class_3/event/pz_bias": 1.5277931690216064, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.855325698852539, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013321113772690296, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17048649489879608, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11186020076274872, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06597644090652466, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1647350788116455, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0034602077212184668, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.41250768303871155, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.023747222498059273, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1899083852767944, "metrics/particle/momentum/class_4/accepted/samples": 289.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24479085206985474, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4363837242126465, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5141586065292358, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.447125792503357, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.096360206604004, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.534393310546875, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07258902490139008, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.4999712705612183, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8163841962814331, "metrics/particle/momentum/class_4/event/energy_bias": -1.3749953508377075, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.926367163658142, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4715195894241333, "metrics/particle/momentum/class_4/event/px_bias": -0.11709104478359222, "metrics/particle/momentum/class_4/event/py_bias": 0.11314449459314346, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.005924105644226, "metrics/particle/momentum/class_4/event/pz_bias": 0.43114912509918213, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5124850273132324, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006522719282656908, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02088644728064537, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.019527453929185867, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00604034960269928, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0507940873503685, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13423782587051392, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.036548539996147156, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4740388095378876, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05273142084479332, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04656209051609039, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04334384202957153, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00017841841327026486, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14270170032978058, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2520753741264343, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.13125281035900116, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2986040115356445, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.2303324043750763, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8176229000091553, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6404455304145813, "metrics/particle/momentum/class_5/event/px_bias": 0.03757447004318237, "metrics/particle/momentum/class_5/event/py_bias": -0.015942897647619247, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.39068588614463806, "metrics/particle/momentum/class_5/event/pz_bias": 0.12775613367557526, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6779558658599854, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5353868007659912, "step_valid_time": 13815.261950016022, "step_total_time": 13816.797337532043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047649541859174, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0728274491790547, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2925, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9599606170003282, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9631215014817254, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9615384615384616, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03687849851827461, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956350508697079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9594995060915377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.957922419460881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9599606170003282, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0049786510945566, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08143013341255623, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2270, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2164, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9530289727831431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9563876651982379, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9547053649956024, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04361233480176212, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9499561018437226, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533039647577093, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.951627088830255, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9530289727831431, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0078431355721953, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07777055932632604, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1316, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9376899696048632, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9369779802581625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06231003039513678, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347496206373292, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9361702127659575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.935459377372817, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936267071320182} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_32000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..dce0157964385af5547ebed3dff03d17ff94a01d --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.515725653656688, "train": {"Classification_binary": 2.4354515075683594, "Classification": 0.0172108206897974, "Regression_pt": 0.11975793540477753, "Regression_eta": 0.012649678625166416, "Regression_sin_phi": 0.0067671677097678185, "Regression_cos_phi": 0.008997933939099312, "Regression_energy": 0.12148769199848175, "Total": 7.684132099151611, "Total_unweighted": 2.7223227359354496}, "train_group": {}, "valid": {"Classification_binary": 2.431490103403727, "Classification": 0.01773679256439209, "Regression_pt": 0.1261388659477234, "Regression_eta": 0.011202028642098108, "Regression_sin_phi": 0.006533695384860039, "Regression_cos_phi": 0.006908737123012543, "Regression_energy": 0.131150354941686, "Total": 7.6539656321207685, "Total_unweighted": 2.7311605780074992, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01764487847685814, "diagnostic/loss/cld_pf/Classification_binary": 2.4383139610290527, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006755426991730928, "diagnostic/loss/cld_pf/Regression_energy": 0.1290324181318283, "diagnostic/loss/cld_pf/Regression_eta": 0.010435718111693859, "diagnostic/loss/cld_pf/Regression_pt": 0.1253354549407959, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006102780811488628, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09726793318986893, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18800152838230133, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07014866173267365, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4155053496360779, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04390564188361168, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18328310549259186, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0710727795958519, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40895751118659973, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9451876282691956, "metrics/particle/class_1/pid_accuracy": 0.9868607521057129, "metrics/particle/class_2/efficiency": 0.45881280303001404, "metrics/particle/class_2/pid_accuracy": 0.7630990743637085, "metrics/particle/class_3/efficiency": 0.7864803075790405, "metrics/particle/class_3/pid_accuracy": 0.986112654209137, "metrics/particle/class_4/efficiency": 0.8163471221923828, "metrics/particle/class_4/pid_accuracy": 0.36588379740715027, "metrics/particle/class_5/efficiency": 0.9786407947540283, "metrics/particle/class_5/pid_accuracy": 0.7083333134651184, "metrics/particle/count/bias_mean": -0.024183006957173347, "metrics/particle/count/mae": 3.128758192062378, "metrics/particle/count/prediction_mean": 70.54117584228516, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05023125186562538, "metrics/particle/event/energy_response_mean": 1.007981538772583, "metrics/particle/event/met_abs_error": 4.982212066650391, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040046609938144684, "metrics/particle/event/scalar_pt_response_mean": 1.0106843709945679, "metrics/particle/event/vector_pt_closure": 0.043023329228162766, "metrics/particle/matched/delta_eta_abs_mean": 0.009421693161129951, "metrics/particle/matched/delta_phi_abs_mean": 0.009533775970339775, "metrics/particle/matched/delta_r_mean": 0.015062857419252396, "metrics/particle/matched/energy_relative_abs_mean": 0.08742942661046982, "metrics/particle/matched/pid_accuracy": 0.9694955945014954, "metrics/particle/matched/pt_relative_abs_mean": 0.08202384412288666, "metrics/particle/matching/duplicate_fraction": 0.030223852023482323, "metrics/particle/matching/efficiency": 0.8377344608306885, "metrics/particle/matching/f1": 0.8378780484199524, "metrics/particle/matching/purity": 0.8380216360092163, "metrics/particle/matching/target_coverage_dr0p05": 0.8235169053077698, "metrics/particle/matching/target_coverage_dr0p10": 0.873375654220581, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7362571358680725, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8719584941864014, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0180098507553339, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08544756472110748, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08089888840913773, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020780134946107864, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13378243148326874, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005268829874694347, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3667212426662445, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.030394980683922768, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4958232641220093, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8351.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2485862672328949, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.904167115688324, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9236862063407898, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8212598562240601, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11430887877941132, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.944304466247559, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01368058379739523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7072886228561401, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.009307287633419037, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6311838626861572, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9868.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8238137364387512, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7548254132270813, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.8379597663879395, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9451181888580322, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2140902578830719, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2046942263841629, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.948823928833008, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.27897393703460693, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.35326862335205, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028286004438996315, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09406144171953201, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09059913456439972, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023667611181735992, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14173530042171478, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018472906202077866, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8566312193870544, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1838243454694748, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.6636149883270264, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1624.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7451694011688232, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3448206186294556, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8615.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8615.7177734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2533125579357147, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 436313.65625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07404520362615585, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.662869930267334, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09902901947498322, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.6483154296875, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2566.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5907602906227112, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.03377991169691086, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.031905651092529, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0270683765411377, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05500401556491852, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09130784869194031, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.885794162750244, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3975259065628052, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.503738403320312, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012684781104326248, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08917352557182312, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08336498588323593, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017785223200917244, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13772915303707123, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00030024017905816436, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10913795977830887, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010756926611065865, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4609612822532654, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19984.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1367959976196289, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6116681098937988, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4610777497291565, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.35827183723449707, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08956548571586609, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.939420700073242, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021747814025729895, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23915870487689972, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005518337246030569, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6455113887786865, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22531.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8740760087966919, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5505280494689941, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.1257004737854, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.1972126960754395, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6412351131439209, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6643310189247131, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.6995511054992676, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.04358436167240143, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.587926864624023, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015014833770692348, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08839987963438034, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08306971192359924, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01893891580402851, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1368623673915863, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026703160256147385, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22145837545394897, "metrics/particle/momentum/all/accepted/pz_bias": -0.025612523779273033, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0723919868469238, "metrics/particle/momentum/all/accepted/samples": 29959.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21299313008785248, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7480230927467346, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 632.8568115234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 632.7501220703125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10856571048498154, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 118198.0703125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0106964111328125, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5491472482681274, "metrics/particle/momentum/all/assigned/pz_bias": -0.006338312290608883, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8269474506378174, "metrics/particle/momentum/all/assigned/samples": 34965.0, "metrics/particle/momentum/all/efficiency": 0.8380374312400818, "metrics/particle/momentum/all/event/energy_bias": 2.3391332626342773, "metrics/particle/momentum/all/event/p3_error_mean": 13.682450294494629, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.895342826843262, "metrics/particle/momentum/all/event/px_bias": 0.4821489155292511, "metrics/particle/momentum/all/event/py_bias": 0.9603331685066223, "metrics/particle/momentum/all/event/pz_abs_mean": 10.173688888549805, "metrics/particle/momentum/all/event/pz_bias": 0.07496759295463562, "metrics/particle/momentum/all/event/pz_residual_rms": 18.735027313232422, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009182492271065712, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04513223096728325, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03735752031207085, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007081281393766403, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08489545434713364, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001049771555699408, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13488885760307312, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.019075803458690643, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8002298474311829, "metrics/particle/momentum/class_1/accepted/samples": 16194.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07299264520406723, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19283254444599152, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21361596882343292, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16552041471004486, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03973625972867012, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.4780161380767822, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004142840392887592, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.26633936166763306, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.004329504445195198, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9483039379119873, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9446421265602112, "metrics/particle/momentum/class_1/event/energy_bias": 2.913295269012451, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.884143829345703, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.949068069458008, "metrics/particle/momentum/class_1/event/px_bias": 0.4563572108745575, "metrics/particle/momentum/class_1/event/py_bias": 0.07997430860996246, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.535794258117676, "metrics/particle/momentum/class_1/event/pz_bias": 0.014627508819103241, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.180562973022461, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03441653773188591, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1911739856004715, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19325542449951172, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.029523396864533424, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23250097036361694, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03978978842496872, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6454089879989624, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.18905800580978394, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.764341354370117, "metrics/particle/momentum/class_2/accepted/samples": 1332.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8338525295257568, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7209876775741577, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 8452.9423828125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 8452.501953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34837475419044495, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 432206.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07533460855484009, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8550360202789307, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.08109205216169357, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.4099321365356445, "metrics/particle/momentum/class_2/assigned/samples": 2615.0, "metrics/particle/momentum/class_2/efficiency": 0.4608996510505676, "metrics/particle/momentum/class_2/event/energy_bias": -6.1900434494018555, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.767844200134277, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.882602214813232, "metrics/particle/momentum/class_2/event/px_bias": 0.0933167040348053, "metrics/particle/momentum/class_2/event/py_bias": 0.4719608426094055, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.14681625366211, "metrics/particle/momentum/class_2/event/pz_bias": -0.9072970747947693, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.3420991897583, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02091657929122448, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13463060557842255, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13283760845661163, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03981984406709671, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17443160712718964, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005842584068886936, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1750437170267105, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.020036548376083374, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6138008236885071, "metrics/particle/momentum/class_3/accepted/samples": 11981.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2654360830783844, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.363271713256836, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3034602403640747, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1858720779418945, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14565883576869965, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.452268600463867, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006945862900465727, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4723648428916931, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0017331874696537852, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.684741735458374, "metrics/particle/momentum/class_3/assigned/samples": 14685.0, "metrics/particle/momentum/class_3/efficiency": 0.7887945175170898, "metrics/particle/momentum/class_3/event/energy_bias": 6.8987321853637695, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.077544212341309, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.986931800842285, "metrics/particle/momentum/class_3/event/px_bias": -0.06557700037956238, "metrics/particle/momentum/class_3/event/py_bias": 0.238381028175354, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.434967994689941, "metrics/particle/momentum/class_3/event/pz_bias": 0.4688161611557007, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.014215469360352, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012079032137989998, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.161708265542984, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10947152972221375, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05436757579445839, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1592164784669876, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010600706562399864, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4664430022239685, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1084100753068924, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4775062799453735, "metrics/particle/momentum/class_4/accepted/samples": 283.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.32837656140327454, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6513571739196777, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.665217399597168, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6013813018798828, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17514124512672424, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.258148193359375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5713655352592468, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10467533767223358, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.684630274772644, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.799435019493103, "metrics/particle/momentum/class_4/event/energy_bias": -1.4861524105072021, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0496034622192383, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5378949642181396, "metrics/particle/momentum/class_4/event/px_bias": -0.05119828134775162, "metrics/particle/momentum/class_4/event/py_bias": 0.18943870067596436, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1032906770706177, "metrics/particle/momentum/class_4/event/pz_bias": 0.41190552711486816, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.348564147949219, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007487561088055372, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02416793629527092, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022449105978012085, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.002518631285056472, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06622398644685745, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17392726242542267, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.016515130177140236, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6750932335853577, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.009585549123585224, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03614945709705353, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03327019512653351, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0017666465137153864, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10247428715229034, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.18236427009105682, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.012304911389946938, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.6741378307342529, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": 0.20330220460891724, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7232584953308105, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5717542171478271, "metrics/particle/momentum/class_5/event/px_bias": 0.04925025254487991, "metrics/particle/momentum/class_5/event/py_bias": -0.019421758130192757, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3455039858818054, "metrics/particle/momentum/class_5/event/pz_bias": 0.086915522813797, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6014994382858276, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.726067304611206, "step_valid_time": 14262.955221652985, "step_total_time": 14264.681289672852, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048706567078391, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0698797844441289, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3054, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2922, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9600523903077931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9611539091952139, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03994760969220694, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.958976042008533, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9567779960707269, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9578757580724472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007048469260948, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0792128407774706, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2180, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2172, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9569798068481123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569798068481123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9569798068481123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04302019315188762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534679543459175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534679543459175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534679543459175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9569798068481123, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0096962443874486, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07624664891285149, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1228, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9311649016641452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9325757575757576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06883509833585477, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9317147192716236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9288956127080181, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9303030303030303, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339908952959028} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_33000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..3ea8665a5ee6930f7b8e353766d4d530b832072a --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5630920803334596, "train": {"Classification_binary": 2.3566534519195557, "Classification": 0.01661510020494461, "Regression_pt": 0.11750560998916626, "Regression_eta": 0.012807735241949558, "Regression_sin_phi": 0.005897498223930597, "Regression_cos_phi": 0.005534461233764887, "Regression_energy": 0.1210373267531395, "Total": 7.254741191864014, "Total_unweighted": 2.636051183566451}, "train_group": {}, "valid": {"Classification_binary": 2.419439951578776, "Classification": 0.01749761775135994, "Regression_pt": 0.12556674083073935, "Regression_eta": 0.011290286978085836, "Regression_sin_phi": 0.006511608759562175, "Regression_cos_phi": 0.006890524178743362, "Regression_energy": 0.13041373093922934, "Total": 7.617547353108724, "Total_unweighted": 2.7176104610164957, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017366832122206688, "diagnostic/loss/cld_pf/Classification_binary": 2.4278597831726074, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006665281485766172, "diagnostic/loss/cld_pf/Regression_energy": 0.12950491905212402, "diagnostic/loss/cld_pf/Regression_eta": 0.010413361713290215, "diagnostic/loss/cld_pf/Regression_pt": 0.12573237717151642, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00617278553545475, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08679676055908203, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18649055063724518, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05967748537659645, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4120916426181793, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.038360703736543655, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18255220353603363, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06552784144878387, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40588247776031494, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9443826675415039, "metrics/particle/class_1/pid_accuracy": 0.9866669178009033, "metrics/particle/class_2/efficiency": 0.47305458784103394, "metrics/particle/class_2/pid_accuracy": 0.7772157788276672, "metrics/particle/class_3/efficiency": 0.7913370132446289, "metrics/particle/class_3/pid_accuracy": 0.9861701130867004, "metrics/particle/class_4/efficiency": 0.8123108148574829, "metrics/particle/class_4/pid_accuracy": 0.4273291826248169, "metrics/particle/class_5/efficiency": 0.9708737730979919, "metrics/particle/class_5/pid_accuracy": 0.6700000166893005, "metrics/particle/count/bias_mean": 0.06143790856003761, "metrics/particle/count/mae": 3.0143790245056152, "metrics/particle/count/prediction_mean": 70.62680053710938, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04787769168615341, "metrics/particle/event/energy_response_mean": 1.0007606744766235, "metrics/particle/event/met_abs_error": 4.723022937774658, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03963855281472206, "metrics/particle/event/scalar_pt_response_mean": 1.011237382888794, "metrics/particle/event/vector_pt_closure": 0.04247483238577843, "metrics/particle/matched/delta_eta_abs_mean": 0.008527347818017006, "metrics/particle/matched/delta_phi_abs_mean": 0.009863384999334812, "metrics/particle/matched/delta_r_mean": 0.014601585455238819, "metrics/particle/matched/energy_relative_abs_mean": 0.08647393435239792, "metrics/particle/matched/pid_accuracy": 0.9701792001724243, "metrics/particle/matched/pt_relative_abs_mean": 0.08218607306480408, "metrics/particle/matching/duplicate_fraction": 0.031010836362838745, "metrics/particle/matching/efficiency": 0.8404760956764221, "metrics/particle/matching/f1": 0.8401103615760803, "metrics/particle/matching/purity": 0.8397449254989624, "metrics/particle/matching/target_coverage_dr0p05": 0.826406717300415, "metrics/particle/matching/target_coverage_dr0p10": 0.8750984072685242, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7384337782859802, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8738665580749512, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016579419374465942, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08272754400968552, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08041515946388245, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01690826751291752, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13297581672668457, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004899617750197649, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.35262057185173035, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.002045049797743559, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.503027319908142, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8368.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23381412029266357, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8488585352897644, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8269208669662476, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7207158207893372, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11469606310129166, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.850703239440918, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013249722309410572, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6793082356452942, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.009854321368038654, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4828901290893555, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9887.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8254907727241516, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.4405176639556885, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.662652492523193, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.998141050338745, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.13926434516906738, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.085646852850914, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.732663631439209, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.26560378074645996, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.440374374389648, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02660699188709259, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09044532477855682, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09061449021100998, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01449176762253046, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13990046083927155, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02434570901095867, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.926749050617218, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15364135801792145, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9618656635284424, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1643.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7554296851158142, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5951594114303589, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5512.20654296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5512.03466796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26325976848602295, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 280009.34375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07975222915410995, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7665083408355713, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.10779974609613419, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.897331237792969, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2583.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5976718664169312, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.6972182989120483, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.798309803009033, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8159413933753967, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.030537746846675873, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1397373527288437, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.672540187835693, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6942048072814941, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.10807991027832, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012754742056131363, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08894200623035431, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0834694430232048, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01659691520035267, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13792119920253754, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005002000834792852, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10972689837217331, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0008708506939001381, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.507195234298706, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19992.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1365344524383545, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6630753874778748, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5101722478866577, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4050895571708679, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0900057703256607, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.338291168212891, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002972097834572196, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23715458810329437, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.012750706635415554, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1687051057815552, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22543.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8744259476661682, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.233265995979309, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.502684593200684, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.380403518676758, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4664827585220337, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.06804794073104858, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0449790954589844, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3627172112464905, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.554628372192383, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014580030925571918, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08729109168052673, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08300885558128357, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016568472608923912, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13667011260986328, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003033030079677701, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22221237421035767, "metrics/particle/momentum/all/accepted/pz_bias": -0.00840368214994669, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1322039365768433, "metrics/particle/momentum/all/accepted/samples": 30003.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20966187119483948, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7842993140220642, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 407.2117919921875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 407.1014404296875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10975923389196396, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 76053.6484375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011538571678102016, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5486072897911072, "metrics/particle/momentum/all/assigned/pz_bias": -0.003039501840248704, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.687239170074463, "metrics/particle/momentum/all/assigned/samples": 35013.0, "metrics/particle/momentum/all/efficiency": 0.8392682075500488, "metrics/particle/momentum/all/event/energy_bias": -0.023434698581695557, "metrics/particle/momentum/all/event/p3_error_mean": 12.774959564208984, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.752097129821777, "metrics/particle/momentum/all/event/px_bias": -0.35775619745254517, "metrics/particle/momentum/all/event/py_bias": 0.15733623504638672, "metrics/particle/momentum/all/event/pz_abs_mean": 9.308497428894043, "metrics/particle/momentum/all/event/pz_bias": 0.06588374078273773, "metrics/particle/momentum/all/event/pz_residual_rms": 16.085670471191406, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008376035839319229, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04437782242894173, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.037646591663360596, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00755486311390996, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08537545800209045, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014229151420295238, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13503484427928925, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00026946960133500397, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8437063694000244, "metrics/particle/momentum/class_1/accepted/samples": 16164.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06722519546747208, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24483810365200043, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.27998003363609314, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23291611671447754, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04223790764808655, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.0796918869018555, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004608832765370607, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2622244954109192, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.00286402041092515, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7077382802963257, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.9428921341896057, "metrics/particle/momentum/class_1/event/energy_bias": 2.05723237991333, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.588003158569336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.996341228485107, "metrics/particle/momentum/class_1/event/px_bias": -0.11639165878295898, "metrics/particle/momentum/class_1/event/py_bias": 0.07909363508224487, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.195714950561523, "metrics/particle/momentum/class_1/event/pz_bias": 0.07089346647262573, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.97058391571045, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0333249568939209, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19028489291667938, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19174844026565552, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04779572784900665, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23047415912151337, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04228486493229866, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.716575026512146, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.169745534658432, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.026196479797363, "metrics/particle/momentum/class_2/accepted/samples": 1348.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8211631774902344, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7436095476150513, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5408.04833984375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5407.58544921875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35676291584968567, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 277390.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07636778056621552, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.952030658721924, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.05827289819717407, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.706750869750977, "metrics/particle/momentum/class_2/assigned/samples": 2632.0, "metrics/particle/momentum/class_2/efficiency": 0.4664359986782074, "metrics/particle/momentum/class_2/event/energy_bias": -6.342441558837891, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.449705123901367, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.827333450317383, "metrics/particle/momentum/class_2/event/px_bias": 0.179692342877388, "metrics/particle/momentum/class_2/event/py_bias": 0.023549258708953857, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.798907279968262, "metrics/particle/momentum/class_2/event/pz_bias": -1.1570979356765747, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.06397533416748, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020992213860154152, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13262081146240234, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13195453584194183, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03502033278346062, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17357851564884186, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007475083111785352, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1675999015569687, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004098866134881973, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6006056070327759, "metrics/particle/momentum/class_3/accepted/samples": 12040.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26687824726104736, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3707906007766724, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.229849100112915, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.106535792350769, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14409026503562927, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.517001152038574, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008088085800409317, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4537491798400879, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0027604044880717993, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3701934814453125, "metrics/particle/momentum/class_3/assigned/samples": 14713.0, "metrics/particle/momentum/class_3/efficiency": 0.7926788926124573, "metrics/particle/momentum/class_3/event/energy_bias": 5.137810230255127, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.696053504943848, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.776284217834473, "metrics/particle/momentum/class_3/event/px_bias": -0.37199902534484863, "metrics/particle/momentum/class_3/event/py_bias": -0.019993968307971954, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.16724157333374, "metrics/particle/momentum/class_3/event/pz_bias": 0.7964308261871338, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.861899375915527, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011820100247859955, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15867161750793457, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11110058426856995, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.059770215302705765, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.166189044713974, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007017544005066156, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.465770423412323, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04270952194929123, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5968232154846191, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.26991868019104004, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.194716691970825, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3217389583587646, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.2735540866851807, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.444416046142578, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6576592326164246, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.05778774619102478, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.185443162918091, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.805084764957428, "metrics/particle/momentum/class_4/event/energy_bias": -0.9545918703079224, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8859281539916992, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3805913925170898, "metrics/particle/momentum/class_4/event/px_bias": -0.13613122701644897, "metrics/particle/momentum/class_4/event/py_bias": 0.07961317896842957, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0530550479888916, "metrics/particle/momentum/class_4/event/pz_bias": 0.22995994985103607, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.12251091003418, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006128894165158272, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019225209951400757, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.018807288259267807, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004436195828020573, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04687315598130226, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11894235014915466, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.05725135654211044, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.44545283913612366, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.029779434204101562, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04248284921050072, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04032259061932564, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00326522090472281, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13028496503829956, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20248249173164368, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1041443794965744, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8114567995071411, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": 0.07855592668056488, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7928202152252197, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6630324721336365, "metrics/particle/momentum/class_5/event/px_bias": 0.08707335591316223, "metrics/particle/momentum/class_5/event/py_bias": -0.0049258507788181305, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.32655587792396545, "metrics/particle/momentum/class_5/event/pz_bias": 0.12569746375083923, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4525548219680786, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.895493507385254, "step_valid_time": 14708.98520231247, "step_total_time": 14710.880696058273, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036083106080929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06732569600921567, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2922, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9682959048877147, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9652674897119341, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.031704095112285335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.958976042008533, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.964993394980185, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9619753086419753, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060820614096215, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07647646217335369, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2169, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552238805970149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9581682078379569, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566937788524951, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041831792162043156, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9521510096575944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9550858652575958, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9536161793800836, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552238805970149, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0101593061803886, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07847784454275752, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9357520786092215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9375236652783037, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06424792139077853, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9327286470143613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9344945096554336, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9393019726858877} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_34000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..66ec6c67b2b475ded40fc86b73da16c77088f6de --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.610458507010231, "train": {"Classification_binary": 2.480783224105835, "Classification": 0.019388308748602867, "Regression_pt": 0.13058727979660034, "Regression_eta": 0.014147646725177765, "Regression_sin_phi": 0.006676904857158661, "Regression_cos_phi": 0.007990559563040733, "Regression_energy": 0.13759875297546387, "Total": 8.133903503417969, "Total_unweighted": 2.797172676771879}, "train_group": {}, "valid": {"Classification_binary": 2.410645325978597, "Classification": 0.017351287106672924, "Regression_pt": 0.1251343091328939, "Regression_eta": 0.011209982136885325, "Regression_sin_phi": 0.006497448931137721, "Regression_cos_phi": 0.00685187429189682, "Regression_energy": 0.13000157475471497, "Total": 7.584698994954427, "Total_unweighted": 2.707691802332799, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017284372821450233, "diagnostic/loss/cld_pf/Classification_binary": 2.414727210998535, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.00663754204288125, "diagnostic/loss/cld_pf/Regression_energy": 0.12928889691829681, "diagnostic/loss/cld_pf/Regression_eta": 0.010344944894313812, "diagnostic/loss/cld_pf/Regression_pt": 0.12535086274147034, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0061489795334637165, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08902937173843384, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18640930950641632, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06191008538007736, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41294369101524353, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04120855778455734, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1824975311756134, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06837569922208786, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40630099177360535, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.946548342704773, "metrics/particle/class_1/pid_accuracy": 0.986717700958252, "metrics/particle/class_2/efficiency": 0.46496525406837463, "metrics/particle/class_2/pid_accuracy": 0.7760353088378906, "metrics/particle/class_3/efficiency": 0.7867000699043274, "metrics/particle/class_3/pid_accuracy": 0.9852505922317505, "metrics/particle/class_4/efficiency": 0.8173562288284302, "metrics/particle/class_4/pid_accuracy": 0.40740740299224854, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.7250996232032776, "metrics/particle/count/bias_mean": -0.14444445073604584, "metrics/particle/count/mae": 3.012418270111084, "metrics/particle/count/prediction_mean": 70.42091369628906, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.048605237156152725, "metrics/particle/event/energy_response_mean": 1.0040793418884277, "metrics/particle/event/met_abs_error": 4.646984100341797, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03963674232363701, "metrics/particle/event/scalar_pt_response_mean": 1.0110275745391846, "metrics/particle/event/vector_pt_closure": 0.04260103777050972, "metrics/particle/matched/delta_eta_abs_mean": 0.008798967115581036, "metrics/particle/matched/delta_phi_abs_mean": 0.008988321758806705, "metrics/particle/matched/delta_r_mean": 0.014165984466671944, "metrics/particle/matched/energy_relative_abs_mean": 0.08609890937805176, "metrics/particle/matched/pid_accuracy": 0.9700154662132263, "metrics/particle/matched/pt_relative_abs_mean": 0.08203929662704468, "metrics/particle/matching/duplicate_fraction": 0.029570091515779495, "metrics/particle/matching/efficiency": 0.8389756083488464, "metrics/particle/matching/f1": 0.8398351669311523, "metrics/particle/matching/purity": 0.8406964540481567, "metrics/particle/matching/target_coverage_dr0p05": 0.8252118825912476, "metrics/particle/matching/target_coverage_dr0p10": 0.8731533288955688, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7361181974411011, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8728476762771606, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015620255842804909, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08254615217447281, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08050865679979324, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01850641332566738, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13310186564922333, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004882114939391613, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3510804772377014, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0037877201102674007, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4957623481750488, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8398.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23441554605960846, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.888678789138794, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8965762853622437, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7921817302703857, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11193198710680008, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.61784839630127, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013561380095779896, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.674373209476471, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -9.81173652689904e-05, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4879016876220703, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9881.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8284502029418945, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.1526973247528076, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.355670928955078, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7466390132904053, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.01902083307504654, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.12203669548034668, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.527449131011963, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.1335645318031311, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.219630241394043, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024844013154506683, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08993039280176163, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09056916832923889, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01903812400996685, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14151246845722198, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024286581203341484, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9440510272979736, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.14545674622058868, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.946676254272461, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1647.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7522437572479248, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6378865242004395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5461.369140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5461.21337890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2590315043926239, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 278440.78125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07840123027563095, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.683881998062134, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.15505217015743256, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.712632179260254, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2602.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.599126935005188, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.46171072125434875, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.107009410858154, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.836549699306488, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05844741314649582, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.14903022348880768, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.970998287200928, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.17811182141304016, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.473063468933105, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012714476324617863, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08854487538337708, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08330380916595459, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01833909936249256, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.138248011469841, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00045105998287908733, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10593411326408386, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006842480506747961, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4677533209323883, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19953.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1358363926410675, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.576701283454895, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.43601858615875244, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.33204853534698486, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08879999816417694, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.9182183742523193, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025333333760499954, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2307526022195816, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.007611442822962999, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1552823781967163, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22500.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8727201223373413, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5801240801811218, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.423076629638672, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.379749298095703, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.43611857295036316, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3296578526496887, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.990360736846924, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.185630202293396, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.025101184844971, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014193911105394363, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08694158494472504, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08292019367218018, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018424319103360176, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13700996339321136, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030002000275999308, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22057901322841644, "metrics/particle/momentum/all/accepted/pz_bias": -0.013597724959254265, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.117403268814087, "metrics/particle/momentum/all/accepted/samples": 29998.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20952798426151276, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7437496185302734, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 406.7447509765625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 406.6368103027344, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10799530893564224, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 75937.7890625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011291198432445526, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5385152101516724, "metrics/particle/momentum/all/assigned/pz_bias": -0.01645578444004059, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6516244411468506, "metrics/particle/momentum/all/assigned/samples": 34983.0, "metrics/particle/momentum/all/efficiency": 0.8391283750534058, "metrics/particle/momentum/all/event/energy_bias": 1.2711106538772583, "metrics/particle/momentum/all/event/p3_error_mean": 13.09108829498291, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.774718284606934, "metrics/particle/momentum/all/event/px_bias": -0.47554513812065125, "metrics/particle/momentum/all/event/py_bias": 0.6007247567176819, "metrics/particle/momentum/all/event/pz_abs_mean": 9.6736421585083, "metrics/particle/momentum/all/event/pz_bias": -0.4973064959049225, "metrics/particle/momentum/all/event/pz_residual_rms": 17.1691951751709, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007851948030292988, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.043762337416410446, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03693467751145363, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006085934117436409, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08405862003564835, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015423530712723732, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13103072345256805, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.004673447925597429, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8342388868331909, "metrics/particle/momentum/class_1/accepted/samples": 16209.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06644957512617111, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23135122656822205, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.24248358607292175, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1935918629169464, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03996266424655914, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.236219644546509, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005075549706816673, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.26112133264541626, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.017747336998581886, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6452499628067017, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.9455171227455139, "metrics/particle/momentum/class_1/event/energy_bias": 2.7885730266571045, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.43271541595459, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.9245147705078125, "metrics/particle/momentum/class_1/event/px_bias": -0.06231109797954559, "metrics/particle/momentum/class_1/event/py_bias": 0.33969929814338684, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.101551532745361, "metrics/particle/momentum/class_1/event/pz_bias": -0.47335511445999146, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.538345336914062, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034405119717121124, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18602240085601807, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18695515394210815, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03752632811665535, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22816705703735352, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03934669494628906, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6878286600112915, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.20703399181365967, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9160754680633545, "metrics/particle/momentum/class_2/accepted/samples": 1347.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.804776668548584, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7023007869720459, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5399.52294921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5399.0751953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3511972725391388, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 276901.96875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07373622059822083, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8334927558898926, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13539521396160126, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.43027400970459, "metrics/particle/momentum/class_2/assigned/samples": 2631.0, "metrics/particle/momentum/class_2/efficiency": 0.46608996391296387, "metrics/particle/momentum/class_2/event/energy_bias": -5.49017333984375, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.43032455444336, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.890941143035889, "metrics/particle/momentum/class_2/event/px_bias": -0.061877600848674774, "metrics/particle/momentum/class_2/event/py_bias": 0.4262152314186096, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.733734130859375, "metrics/particle/momentum/class_2/event/pz_bias": -1.6253703832626343, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.194392204284668, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020675862208008766, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13364778459072113, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1336951106786728, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04057801514863968, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17578986287117004, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009175077429972589, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17438577115535736, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006341525353491306, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6487343907356262, "metrics/particle/momentum/class_3/accepted/samples": 11989.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2700165808200836, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.304524302482605, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2341030836105347, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1157183647155762, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14362572133541107, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.748113632202148, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007354944013059139, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4515281319618225, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.003718764055520296, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.455559253692627, "metrics/particle/momentum/class_3/assigned/samples": 14684.0, "metrics/particle/momentum/class_3/efficiency": 0.78932124376297, "metrics/particle/momentum/class_3/event/energy_bias": 4.987252235412598, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.60260009765625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.699554443359375, "metrics/particle/momentum/class_3/event/px_bias": -0.2104034274816513, "metrics/particle/momentum/class_3/event/py_bias": -0.2207116037607193, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.121123313903809, "metrics/particle/momentum/class_3/event/pz_bias": 1.087233543395996, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.879776000976562, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011812269687652588, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1484876573085785, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1077943816781044, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0585186630487442, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1618378907442093, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003496503457427025, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3890288770198822, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06363818049430847, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2462043762207031, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2997805178165436, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.9448041915893555, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0448694229125977, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9922648668289185, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.113943099975586, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.694104790687561, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0969948098063469, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.4475128650665283, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8079096078872681, "metrics/particle/momentum/class_4/event/energy_bias": -1.2430378198623657, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7486286163330078, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3140254020690918, "metrics/particle/momentum/class_4/event/px_bias": -0.2636183202266693, "metrics/particle/momentum/class_4/event/py_bias": 0.0640183538198471, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9289711117744446, "metrics/particle/momentum/class_4/event/pz_bias": 0.4174787402153015, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.297224521636963, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0054597510024905205, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02028193511068821, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01938537135720253, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008189222775399685, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05378146469593048, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.10523288697004318, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.027246668934822083, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4061587154865265, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.014404604211449623, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04135715216398239, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.038898058235645294, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01061802264302969, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1268150508403778, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.18568333983421326, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.02418067678809166, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8393300175666809, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.2284957766532898, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.6959385871887207, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5743496417999268, "metrics/particle/momentum/class_5/event/px_bias": 0.12266527861356735, "metrics/particle/momentum/class_5/event/py_bias": -0.008496538735926151, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.30222687125205994, "metrics/particle/momentum/class_5/event/pz_bias": 0.09670673310756683, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4717812538146973, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6775197982788086, "step_valid_time": 15152.166247606277, "step_total_time": 15153.843767642975, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031865350790956, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0708392399745883, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9638395792241946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.963048119559862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03616042077580539, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9586478503446012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.960223537146614, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9594350468057152, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622579586478504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070499269004147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07742142177589639, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2179, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.956540825285338, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569609134826527, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9567508232711306, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043039086517347384, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552238805970149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9556433904259991, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554335894621295, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.956540825285338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0119938872020668, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07438253144119256, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1320, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9348484848484848, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.935557240333586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06515151515151515, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9340909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9347990902198635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936267071320182} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_35000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..2294b0414d5560a4d553fda5f07afa987eaf12a7 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6578249336870026, "train": {"Classification_binary": 2.420729637145996, "Classification": 0.017168741673231125, "Regression_pt": 0.12428867071866989, "Regression_eta": 0.011500132270157337, "Regression_sin_phi": 0.007292799185961485, "Regression_cos_phi": 0.00772916991263628, "Regression_energy": 0.13003525137901306, "Total": 7.702817916870117, "Total_unweighted": 2.7187444022856653}, "train_group": {}, "valid": {"Classification_binary": 2.4016950925191245, "Classification": 0.01747124269604683, "Regression_pt": 0.12496484319368999, "Regression_eta": 0.011218545337518057, "Regression_sin_phi": 0.00648733414709568, "Regression_cos_phi": 0.006899502749244372, "Regression_energy": 0.12995566924413046, "Total": 7.5819142659505205, "Total_unweighted": 2.69869222988685, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017335623502731323, "diagnostic/loss/cld_pf/Classification_binary": 2.406099319458008, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006704540457576513, "diagnostic/loss/cld_pf/Regression_energy": 0.1282854527235031, "diagnostic/loss/cld_pf/Regression_eta": 0.01034066453576088, "diagnostic/loss/cld_pf/Regression_pt": 0.12445530295372009, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006074862089008093, "diagnostic/regression/cld_pf/energy_pred_mean": 0.07554759830236435, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18306075036525726, "diagnostic/regression/cld_pf/energy_residual_mean": 0.04842832311987877, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4074673652648926, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.02720903791487217, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17905567586421967, "diagnostic/regression/cld_pf/pt_residual_mean": 0.054376181215047836, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4003565013408661, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9486564993858337, "metrics/particle/class_1/pid_accuracy": 0.987171471118927, "metrics/particle/class_2/efficiency": 0.4691808223724365, "metrics/particle/class_2/pid_accuracy": 0.7824186682701111, "metrics/particle/class_3/efficiency": 0.79652339220047, "metrics/particle/class_3/pid_accuracy": 0.9854324460029602, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.39853301644325256, "metrics/particle/class_5/efficiency": 0.9805825352668762, "metrics/particle/class_5/pid_accuracy": 0.6594059467315674, "metrics/particle/count/bias_mean": 0.013071895577013493, "metrics/particle/count/mae": 3.113725423812866, "metrics/particle/count/prediction_mean": 70.57843017578125, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04802108556032181, "metrics/particle/event/energy_response_mean": 0.9825170040130615, "metrics/particle/event/met_abs_error": 4.479578495025635, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03730889782309532, "metrics/particle/event/scalar_pt_response_mean": 0.9945826530456543, "metrics/particle/event/vector_pt_closure": 0.041216060519218445, "metrics/particle/matched/delta_eta_abs_mean": 0.008969541639089584, "metrics/particle/matched/delta_phi_abs_mean": 0.009036830626428127, "metrics/particle/matched/delta_r_mean": 0.014279576018452644, "metrics/particle/matched/energy_relative_abs_mean": 0.08549504727125168, "metrics/particle/matched/pid_accuracy": 0.9701376557350159, "metrics/particle/matched/pt_relative_abs_mean": 0.08167761564254761, "metrics/particle/matching/duplicate_fraction": 0.028216881677508354, "metrics/particle/matching/efficiency": 0.84457927942276, "metrics/particle/matching/f1": 0.8445010185241699, "metrics/particle/matching/purity": 0.8444228172302246, "metrics/particle/matching/target_coverage_dr0p05": 0.8295280933380127, "metrics/particle/matching/target_coverage_dr0p10": 0.8758115768432617, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7434631586074829, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8784605860710144, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015843253582715988, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08090712875127792, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07869016379117966, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012364767491817474, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13031765818595886, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004664513748139143, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3541955053806305, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008990970440208912, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5379712581634521, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8361.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.240940123796463, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9922701120376587, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9536190032958984, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8435884714126587, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11086317151784897, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 23.2635498046875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012983061373233795, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6759747266769409, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.021245192736387253, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6813511848449707, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9859.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8248002529144287, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.249833345413208, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.5049543380737305, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8069992065429688, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.01962999254465103, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.17676569521427155, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.648571014404297, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5241999626159668, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.926494598388672, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02380530722439289, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0949302464723587, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09346535056829453, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.008919717743992805, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14655457437038422, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022560976445674896, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9987207651138306, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13257649540901184, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2211196422576904, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1640.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7288129329681396, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2417540550231934, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5524.7626953125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5524.5986328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2446892261505127, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 278425.71875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07159716635942459, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.564068078994751, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.03673513978719711, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.793126106262207, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2542.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5965805649757385, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.974602460861206, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.3992767333984375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8946472406387329, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04880394786596298, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11807186901569366, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.267481803894043, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.1761816143989563, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.59178352355957, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012603335082530975, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08806867897510529, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08336111903190613, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010674803517758846, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13795509934425354, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000495000509545207, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10893061012029648, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0016487442189827561, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.498096227645874, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20202.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13077513873577118, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5865601897239685, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4311138689517975, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3198794722557068, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08169688284397125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.375885486602783, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021277538035064936, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22342094779014587, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007047964725643396, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4958707094192505, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22559.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8836110830307007, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.42410776019096375, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.443596839904785, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.607385635375977, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.46365514397621155, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.028486065566539764, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.4123902320861816, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.02835255116224289, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.171571254730225, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014108489267528057, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08645874261856079, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08261672407388687, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011047330684959888, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13637179136276245, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002847399329766631, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22514145076274872, "metrics/particle/momentum/all/accepted/pz_bias": -0.010790547356009483, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1764901876449585, "metrics/particle/momentum/all/accepted/samples": 30203.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2053268700838089, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7486139535903931, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 402.2618713378906, "metrics/particle/momentum/all/assigned/pt_relative_bias": 402.1470947265625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1017734557390213, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 75077.7734375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010240274481475353, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5212371349334717, "metrics/particle/momentum/all/assigned/pz_bias": -0.004114474635571241, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6128201484680176, "metrics/particle/momentum/all/assigned/samples": 34960.0, "metrics/particle/momentum/all/efficiency": 0.8448628187179565, "metrics/particle/momentum/all/event/energy_bias": -5.648543357849121, "metrics/particle/momentum/all/event/p3_error_mean": 12.119803428649902, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.353377342224121, "metrics/particle/momentum/all/event/px_bias": -0.532089114189148, "metrics/particle/momentum/all/event/py_bias": -0.08717990666627884, "metrics/particle/momentum/all/event/pz_abs_mean": 8.868461608886719, "metrics/particle/momentum/all/event/pz_bias": -0.31966590881347656, "metrics/particle/momentum/all/event/pz_residual_rms": 16.01816177368164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008396895602345467, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04339628294110298, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.036955032497644424, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0032265407498925924, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08427577465772629, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014148622285574675, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1391984075307846, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.009326618164777756, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8903540372848511, "metrics/particle/momentum/class_1/accepted/samples": 16256.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06789711862802505, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18876923620700836, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.20916211605072021, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15768739581108093, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03593512997031212, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6110897064208984, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004200209863483906, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2488497942686081, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.022246723994612694, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.5368530750274658, "metrics/particle/momentum/class_1/assigned/samples": 17142.0, "metrics/particle/momentum/class_1/efficiency": 0.9482587575912476, "metrics/particle/momentum/class_1/event/energy_bias": -0.11239345371723175, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.062644958496094, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.671933650970459, "metrics/particle/momentum/class_1/event/px_bias": -0.1219148337841034, "metrics/particle/momentum/class_1/event/py_bias": -0.21394476294517517, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.781239986419678, "metrics/particle/momentum/class_1/event/pz_bias": -0.581703782081604, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.006233215332031, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03257961943745613, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1952590048313141, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19635722041130066, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.040195416659116745, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23642860352993011, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.037009622901678085, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7341666221618652, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14216017723083496, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.076083183288574, "metrics/particle/momentum/class_2/accepted/samples": 1351.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.805422842502594, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.634983241558075, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5393.72119140625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5393.2578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.351901650428772, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 275144.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06991932541131973, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.864410877227783, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.042624570429325104, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.638902187347412, "metrics/particle/momentum/class_2/assigned/samples": 2603.0, "metrics/particle/momentum/class_2/efficiency": 0.4674740433692932, "metrics/particle/momentum/class_2/event/energy_bias": -7.8501996994018555, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.07197380065918, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.60298490524292, "metrics/particle/momentum/class_2/event/px_bias": -0.22003167867660522, "metrics/particle/momentum/class_2/event/py_bias": 0.28220123052597046, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.567951202392578, "metrics/particle/momentum/class_2/event/pz_bias": -0.7816312313079834, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.174206733703613, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01985258236527443, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13109809160232544, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13128355145454407, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02638857066631317, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17276057600975037, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009059463045559824, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16874881088733673, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0010445495136082172, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6900403499603271, "metrics/particle/momentum/class_3/accepted/samples": 12142.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26053282618522644, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3621976375579834, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2879014015197754, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1591142416000366, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13371460139751434, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.664058685302734, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006672113202512264, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.42130836844444275, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01952502131462097, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.325359344482422, "metrics/particle/momentum/class_3/assigned/samples": 14688.0, "metrics/particle/momentum/class_3/efficiency": 0.7993943095207214, "metrics/particle/momentum/class_3/event/energy_bias": 3.4318666458129883, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.635292053222656, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.717787742614746, "metrics/particle/momentum/class_3/event/px_bias": -0.0790882408618927, "metrics/particle/momentum/class_3/event/py_bias": -0.23357030749320984, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.1239800453186035, "metrics/particle/momentum/class_3/event/pz_bias": 0.7066128253936768, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.243203163146973, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012284012511372566, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16307279467582703, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11044218391180038, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0480399914085865, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16357240080833435, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007017544005066156, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4339248538017273, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030663052573800087, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5222718715667725, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2535749077796936, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.5840084552764893, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0105152130126953, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9305202960968018, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1666666716337204, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.56850528717041, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7913685441017151, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15697166323661804, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.323254346847534, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.805084764957428, "metrics/particle/momentum/class_4/event/energy_bias": -1.133626103401184, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8028205633163452, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3242989778518677, "metrics/particle/momentum/class_4/event/px_bias": -0.19634288549423218, "metrics/particle/momentum/class_4/event/py_bias": 0.07526800036430359, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9997230172157288, "metrics/particle/momentum/class_4/event/pz_bias": 0.26826900243759155, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.034180164337158, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006229058373719454, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022484812885522842, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02208603173494339, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008369679562747478, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06433148682117462, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12817952036857605, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.021638305857777596, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4226618707180023, "metrics/particle/momentum/class_5/accepted/samples": 169.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007788155693560839, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03526289761066437, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.033306218683719635, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0048285420052707195, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1042325422167778, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.1866072565317154, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.03532695397734642, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8283517360687256, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9768785834312439, "metrics/particle/momentum/class_5/event/energy_bias": 0.01580946147441864, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7308390140533447, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6174676418304443, "metrics/particle/momentum/class_5/event/px_bias": 0.08528851717710495, "metrics/particle/momentum/class_5/event/py_bias": 0.002865951508283615, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.2956431210041046, "metrics/particle/momentum/class_5/event/pz_bias": 0.06878729909658432, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.3663208484649658, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.847968578338623, "step_valid_time": 15598.587574720383, "step_total_time": 15600.43554353714, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982476270962036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06776212731551379, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2926, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9632425336396455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9667325428194994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9649843827059017, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033267457180500656, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602888086642599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9637681159420289, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9620253164556962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9632425336396455, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9976248874039826, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07631453267822808, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2168, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9594177326863697, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9570957095709571, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04058226731363035, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.95171202809482, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9563299514777238, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.954015401540154, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547848990342406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996469629110709, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0709201771567396, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9385898407884761, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9389457717102768, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06141015921152388, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9370735405610311, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9374288964732651, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9393019726858877} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_36000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..d598d604a7a43c5e5410d271c8ed995c6583e778 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.705191360363774, "train": {"Classification_binary": 2.549160957336426, "Classification": 0.018530726432800293, "Regression_pt": 0.12856167554855347, "Regression_eta": 0.01839209720492363, "Regression_sin_phi": 0.006085480563342571, "Regression_cos_phi": 0.007748634088784456, "Regression_energy": 0.13585707545280457, "Total": 8.273385047912598, "Total_unweighted": 2.8643366466276348}, "train_group": {}, "valid": {"Classification_binary": 2.389655113220215, "Classification": 0.0173927607635657, "Regression_pt": 0.12331587076187134, "Regression_eta": 0.011047185709079107, "Regression_sin_phi": 0.006448918953537941, "Regression_cos_phi": 0.006812262659271558, "Regression_energy": 0.12816674510637918, "Total": 7.509864171346028, "Total_unweighted": 2.6828388571739197, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017263878136873245, "diagnostic/loss/cld_pf/Classification_binary": 2.390523672103882, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006676914636045694, "diagnostic/loss/cld_pf/Regression_energy": 0.1262037605047226, "diagnostic/loss/cld_pf/Regression_eta": 0.010231710970401764, "diagnostic/loss/cld_pf/Regression_pt": 0.12255395948886871, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006082569714635611, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0868551954627037, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18535710871219635, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05973590910434723, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41207975149154663, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03338365629315376, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.18022099137306213, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06055079400539398, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40505450963974, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9476215839385986, "metrics/particle/class_1/pid_accuracy": 0.9869349598884583, "metrics/particle/class_2/efficiency": 0.47271278500556946, "metrics/particle/class_2/pid_accuracy": 0.7869366407394409, "metrics/particle/class_3/efficiency": 0.7936225533485413, "metrics/particle/class_3/pid_accuracy": 0.9857668876647949, "metrics/particle/class_4/efficiency": 0.8395559787750244, "metrics/particle/class_4/pid_accuracy": 0.3858173191547394, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6759443283081055, "metrics/particle/count/bias_mean": -0.03790849819779396, "metrics/particle/count/mae": 3.0104575157165527, "metrics/particle/count/prediction_mean": 70.52745056152344, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04734554886817932, "metrics/particle/event/energy_response_mean": 1.0007728338241577, "metrics/particle/event/met_abs_error": 4.799493312835693, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037839796394109726, "metrics/particle/event/scalar_pt_response_mean": 1.006544589996338, "metrics/particle/event/vector_pt_closure": 0.042415693402290344, "metrics/particle/matched/delta_eta_abs_mean": 0.0076256999745965, "metrics/particle/matched/delta_phi_abs_mean": 0.008983581326901913, "metrics/particle/matched/delta_r_mean": 0.01324070431292057, "metrics/particle/matched/energy_relative_abs_mean": 0.086765356361866, "metrics/particle/matched/pid_accuracy": 0.9701456427574158, "metrics/particle/matched/pt_relative_abs_mean": 0.08121735602617264, "metrics/particle/matching/duplicate_fraction": 0.029080597683787346, "metrics/particle/matching/efficiency": 0.8432547450065613, "metrics/particle/matching/f1": 0.8434813022613525, "metrics/particle/matching/purity": 0.8437079787254333, "metrics/particle/matching/target_coverage_dr0p05": 0.8290371894836426, "metrics/particle/matching/target_coverage_dr0p10": 0.8747464418411255, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7425369620323181, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8764784932136536, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015512180514633656, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.083135224878788, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07907336950302124, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013178162276744843, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1308615654706955, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004885605536401272, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.35282525420188904, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.009270585142076015, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4999737739562988, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8392.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23929472267627716, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9259846210479736, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8840636014938354, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7762274742126465, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11202185600996017, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.104318618774414, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013661202043294907, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.697830319404602, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.028994077816605568, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.701000928878784, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9882.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8278583288192749, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.29343801736831665, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.171621799468994, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.718766212463379, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.14535659551620483, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.11882245540618896, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.343160152435303, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.008632302284240723, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.219588279724121, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024310242384672165, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09391394257545471, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0913669541478157, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01782665215432644, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14271298050880432, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02053140103816986, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9079663753509521, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15135285258293152, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9193108081817627, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1656.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7369813919067383, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.364677906036377, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 16821.287109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 16821.123046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2589147388935089, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 854271.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07364340871572495, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5180466175079346, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.17859050631523132, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.336288928985596, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2580.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6024008989334106, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.25881674885749817, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.036929607391357, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8335951566696167, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.047941938042640686, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11900287121534348, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.917879104614258, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.46343350410461426, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.797182083129883, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011280518025159836, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08880630135536194, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08236880600452423, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009854747913777828, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13643862307071686, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003980495675932616, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1045943945646286, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0019119343487545848, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4756518006324768, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20098.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13522639870643616, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6171861290931702, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4640777111053467, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.35466739535331726, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08405076712369919, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.7951788902282715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00252951099537313, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23251375555992126, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.011086740531027317, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5087884664535522, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22534.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8790622353553772, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.9311614632606506, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.048612594604492, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.199078559875488, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.029080307111144066, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.06782831251621246, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.54683518409729, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.011001449078321457, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.230744361877441, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01317427959293127, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0875081792473793, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08194571733474731, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011217832565307617, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13526538014411926, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002753267530351877, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2178279310464859, "metrics/particle/momentum/all/accepted/pz_bias": -0.00445881811901927, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.115939736366272, "metrics/particle/momentum/all/accepted/samples": 30146.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20897571742534637, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7594902515411377, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1240.6595458984375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1240.5462646484375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10484055429697037, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 231951.28125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010915533639490604, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5324032306671143, "metrics/particle/momentum/all/assigned/pz_bias": 0.0021598052699118853, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5467052459716797, "metrics/particle/momentum/all/assigned/samples": 34996.0, "metrics/particle/momentum/all/efficiency": 0.8432683348655701, "metrics/particle/momentum/all/event/energy_bias": 0.9657827615737915, "metrics/particle/momentum/all/event/p3_error_mean": 12.868776321411133, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.893515110015869, "metrics/particle/momentum/all/event/px_bias": -0.22237882018089294, "metrics/particle/momentum/all/event/py_bias": 0.3056536614894867, "metrics/particle/momentum/all/event/pz_abs_mean": 9.20960807800293, "metrics/particle/momentum/all/event/pz_bias": 0.4437997341156006, "metrics/particle/momentum/all/event/pz_residual_rms": 16.018177032470703, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007787008304148912, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04503599554300308, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03654812276363373, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005983063951134682, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08341576159000397, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012328935554251075, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12732449173927307, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007081776857376099, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7955926060676575, "metrics/particle/momentum/class_1/accepted/samples": 16222.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06866566091775894, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2105850875377655, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22798478603363037, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.179155170917511, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0388542078435421, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.459052562713623, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004317134153097868, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2512320280075073, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.011576351709663868, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.4821003675460815, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.9462754726409912, "metrics/particle/momentum/class_1/event/energy_bias": 2.673962354660034, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.484724044799805, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.122091293334961, "metrics/particle/momentum/class_1/event/px_bias": 0.48623189330101013, "metrics/particle/momentum/class_1/event/py_bias": 0.01995539665222168, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.8786468505859375, "metrics/particle/momentum/class_1/event/pz_bias": -0.587608814239502, "metrics/particle/momentum/class_1/event/pz_residual_rms": 9.8870849609375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032896243035793304, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18967799842357635, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1918868124485016, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05071462318301201, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23115503787994385, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.038601603358983994, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6884323358535767, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06958641856908798, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9366188049316406, "metrics/particle/momentum/class_2/accepted/samples": 1373.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8030427098274231, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7002776861190796, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 16493.4765625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 16493.015625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3485366702079773, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 845951.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07563664019107819, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.84253191947937, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07053519040346146, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.342170238494873, "metrics/particle/momentum/class_2/assigned/samples": 2631.0, "metrics/particle/momentum/class_2/efficiency": 0.475086510181427, "metrics/particle/momentum/class_2/event/energy_bias": -5.666456699371338, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.144668579101562, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.664022445678711, "metrics/particle/momentum/class_2/event/px_bias": -0.473112553358078, "metrics/particle/momentum/class_2/event/py_bias": 0.5758134126663208, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.565776824951172, "metrics/particle/momentum/class_2/event/pz_bias": -0.8731229305267334, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.082900047302246, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018300551921129227, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13203011453151703, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1306198239326477, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024311715736985207, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17186520993709564, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006617586477659643, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16798706352710724, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.003602366428822279, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6603676080703735, "metrics/particle/momentum/class_3/accepted/samples": 12089.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26768818497657776, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3817723989486694, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3175888061523438, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1893181800842285, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13825951516628265, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.849212646484375, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007144315168261528, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4495428502559662, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02871410734951496, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3624470233917236, "metrics/particle/momentum/class_3/assigned/samples": 14697.0, "metrics/particle/momentum/class_3/efficiency": 0.7959049344062805, "metrics/particle/momentum/class_3/event/energy_bias": 5.003312110900879, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.672393798828125, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.787628650665283, "metrics/particle/momentum/class_3/event/px_bias": -0.2661363184452057, "metrics/particle/momentum/class_3/event/py_bias": -0.40428435802459717, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.121406555175781, "metrics/particle/momentum/class_3/event/pz_bias": 1.4807525873184204, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.176513671875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011825669556856155, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16057473421096802, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10719466209411621, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05391902104020119, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1620819866657257, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006779660936444998, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4423380196094513, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0841243639588356, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5195238590240479, "metrics/particle/momentum/class_4/accepted/samples": 295.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.23784352838993073, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.293715238571167, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4144014120101929, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3558392524719238, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.781180381774902, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6002797484397888, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08256825059652328, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9992672204971313, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8333333134651184, "metrics/particle/momentum/class_4/event/energy_bias": -1.1461451053619385, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9543581008911133, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4430445432662964, "metrics/particle/momentum/class_4/event/px_bias": -0.08938848227262497, "metrics/particle/momentum/class_4/event/py_bias": 0.1288108378648758, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0784692764282227, "metrics/particle/momentum/class_4/event/pz_bias": 0.3346638083457947, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.273212432861328, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005631582345813513, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02118627540767193, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.019795866683125496, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005606388673186302, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05492251738905907, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12981754541397095, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04575754329562187, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5082685947418213, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02948031760752201, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04143991693854332, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03797558695077896, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0058987559750676155, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12422341108322144, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.15886558592319489, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.048280563205480576, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.5549028515815735, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.10110983997583389, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.828721284866333, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6767556667327881, "metrics/particle/momentum/class_5/event/px_bias": 0.12002666294574738, "metrics/particle/momentum/class_5/event/py_bias": -0.014641585759818554, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.36236652731895447, "metrics/particle/momentum/class_5/event/pz_bias": 0.08911509811878204, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6309705972671509, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6607425212860107, "step_valid_time": 16043.13094162941, "step_total_time": 16044.791684627533, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002236360266322, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06653950084555038, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625861503117821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9648026315789474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9636931164777395, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03519736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9586478503446012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9608552631578947, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9597502874979464, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625861503117821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024774809162182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07512792815350888, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2272, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578577699736611, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.960387323943662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591208791208791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03961267605633803, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9568661971830986, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556043956043956, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578577699736611, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00474554340955, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07457413598188334, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1244, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1240, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9438543247344461, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9445709946848899, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9442125237191651, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0554290053151101, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9408194233687405, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9415337889141989, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9411764705882353, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9438543247344461} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_37000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..8ed857f442573e3ff532d8134fe6283e623718af --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7525577870405458, "train": {"Classification_binary": 2.407191753387451, "Classification": 0.016652634367346764, "Regression_pt": 0.12891912460327148, "Regression_eta": 0.011368803679943085, "Regression_sin_phi": 0.007140773814171553, "Regression_cos_phi": 0.006811338011175394, "Regression_energy": 0.13454864919185638, "Total": 7.689639568328857, "Total_unweighted": 2.712633077055216}, "train_group": {}, "valid": {"Classification_binary": 2.397446632385254, "Classification": 0.01738680029908816, "Regression_pt": 0.12463817993799846, "Regression_eta": 0.010979915658632914, "Regression_sin_phi": 0.006400703762968381, "Regression_cos_phi": 0.006844368452827136, "Regression_energy": 0.12916017572085062, "Total": 7.540028254191081, "Total_unweighted": 2.6928567762176194, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.0172362569719553, "diagnostic/loss/cld_pf/Classification_binary": 2.3952853679656982, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0065554408356547356, "diagnostic/loss/cld_pf/Regression_energy": 0.12600252032279968, "diagnostic/loss/cld_pf/Regression_eta": 0.010367760434746742, "diagnostic/loss/cld_pf/Regression_pt": 0.1224655956029892, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005968825425952673, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08931642025709152, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18429167568683624, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06219712644815445, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4113137125968933, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03350121155381203, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17864976823329926, "diagnostic/regression/cld_pf/pt_residual_mean": 0.060668352991342545, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40253546833992004, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9486948251724243, "metrics/particle/class_1/pid_accuracy": 0.9862427115440369, "metrics/particle/class_2/efficiency": 0.46268656849861145, "metrics/particle/class_2/pid_accuracy": 0.7860133051872253, "metrics/particle/class_3/efficiency": 0.7888317704200745, "metrics/particle/class_3/pid_accuracy": 0.9852347373962402, "metrics/particle/class_4/efficiency": 0.8557013273239136, "metrics/particle/class_4/pid_accuracy": 0.3820754587650299, "metrics/particle/class_5/efficiency": 0.9825242757797241, "metrics/particle/class_5/pid_accuracy": 0.6897233128547668, "metrics/particle/count/bias_mean": -0.33529412746429443, "metrics/particle/count/mae": 3.0241830348968506, "metrics/particle/count/prediction_mean": 70.23006439208984, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.046878885477781296, "metrics/particle/event/energy_response_mean": 0.9954026341438293, "metrics/particle/event/met_abs_error": 4.516807556152344, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037712983787059784, "metrics/particle/event/scalar_pt_response_mean": 1.0012255907058716, "metrics/particle/event/vector_pt_closure": 0.04112563282251358, "metrics/particle/matched/delta_eta_abs_mean": 0.008926008827984333, "metrics/particle/matched/delta_phi_abs_mean": 0.009223699569702148, "metrics/particle/matched/delta_r_mean": 0.014455207623541355, "metrics/particle/matched/energy_relative_abs_mean": 0.08631760627031326, "metrics/particle/matched/pid_accuracy": 0.9695962071418762, "metrics/particle/matched/pt_relative_abs_mean": 0.08068418502807617, "metrics/particle/matching/duplicate_fraction": 0.02810557186603546, "metrics/particle/matching/efficiency": 0.8411151766777039, "metrics/particle/matching/f1": 0.8431182503700256, "metrics/particle/matching/purity": 0.8451308608055115, "metrics/particle/matching/target_coverage_dr0p05": 0.8235446810722351, "metrics/particle/matching/target_coverage_dr0p10": 0.8735793828964233, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7419719099998474, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8748853802680969, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01687602512538433, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08226360380649567, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07857783883810043, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01140849944204092, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13061189651489258, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005004766397178173, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3539907932281494, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.005760791711509228, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5080188512802124, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8392.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23778091371059418, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.867581844329834, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.806844174861908, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.695745587348938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11058083176612854, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.558162689208984, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014520714990794659, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6793420314788818, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.002676245756447315, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.480496406555176, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9848.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8278583288192749, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3934113681316376, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.921839237213135, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.581777334213257, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.28053000569343567, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.011343153193593025, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.131255149841309, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.04909190535545349, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.42291259765625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026169566437602043, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08901803195476532, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08729106932878494, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012075459584593773, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13563577830791473, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02166065014898777, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8857854604721069, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06104360148310661, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8614635467529297, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1662.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.718674898147583, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.642763614654541, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17586.6796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17586.52734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24617797136306763, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 888118.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07291258126497269, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.597816228866577, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.189907968044281, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.302858352661133, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2551.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6045835018157959, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1917771100997925, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.191166400909424, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8644422888755798, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0968678891658783, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09385207295417786, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.05871057510376, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.42422187328338623, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.148117065429688, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01244294736534357, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08910277485847473, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08244550973176956, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015463941730558872, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13695113360881805, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00039964032475836575, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10536559671163559, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0009830098133534193, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4678884744644165, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20018.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1363103687763214, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5919852256774902, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4554266333580017, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.34966427087783813, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08714190870523453, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.663956642150879, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002353986259549856, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.228735089302063, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0016159972874447703, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1064972877502441, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22515.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8755631446838379, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.070813536643982, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.120461940765381, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.177976608276367, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.26789605617523193, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.11863601207733154, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.7121589183807373, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.47575944662094116, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.732283592224121, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01443869061768055, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08718952536582947, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08163398504257202, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014144943095743656, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13513867557048798, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028598031494766474, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21787956357002258, "metrics/particle/momentum/all/accepted/pz_bias": -0.0011117324465885758, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1103512048721313, "metrics/particle/momentum/all/accepted/samples": 30072.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2074822336435318, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7464966177940369, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1285.4964599609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1285.3856201171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10537320375442505, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 240063.484375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010941170156002045, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5289328098297119, "metrics/particle/momentum/all/assigned/pz_bias": 0.014162907376885414, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7499046325683594, "metrics/particle/momentum/all/assigned/samples": 34914.0, "metrics/particle/momentum/all/efficiency": 0.8411983847618103, "metrics/particle/momentum/all/event/energy_bias": -0.5143749713897705, "metrics/particle/momentum/all/event/p3_error_mean": 12.824827194213867, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.814624309539795, "metrics/particle/momentum/all/event/px_bias": -0.6452938914299011, "metrics/particle/momentum/all/event/py_bias": 0.22383123636245728, "metrics/particle/momentum/all/event/pz_abs_mean": 9.202747344970703, "metrics/particle/momentum/all/event/pz_bias": 0.8508894443511963, "metrics/particle/momentum/all/event/pz_residual_rms": 18.094924926757812, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00784353632479906, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.043763697147369385, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03578060120344162, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0065838368609547615, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08176569640636444, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012925462797284126, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12584653496742249, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0009144139476120472, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7784357666969299, "metrics/particle/momentum/class_1/accepted/samples": 16247.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0670894980430603, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20364230871200562, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21892309188842773, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17172446846961975, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.036704208701848984, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.5020344257354736, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003968022298067808, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24406200647354126, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008480108343064785, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6615248918533325, "metrics/particle/momentum/class_1/assigned/samples": 17137.0, "metrics/particle/momentum/class_1/efficiency": 0.9477337598800659, "metrics/particle/momentum/class_1/event/energy_bias": 2.1686649322509766, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.236109733581543, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.731584072113037, "metrics/particle/momentum/class_1/event/px_bias": 0.18762126564979553, "metrics/particle/momentum/class_1/event/py_bias": 0.11552366614341736, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.015866279602051, "metrics/particle/momentum/class_1/event/pz_bias": 0.11609691381454468, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.13166332244873, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03407958149909973, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.190892294049263, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19246232509613037, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03714735060930252, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2324761152267456, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04026845470070839, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7180275917053223, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.09432137757539749, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.0269927978515625, "metrics/particle/momentum/class_2/accepted/samples": 1341.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8095945119857788, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6895114183425903, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17253.609375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17253.1640625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3480769097805023, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 879709.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07384615391492844, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7640819549560547, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.05890556797385216, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.104867458343506, "metrics/particle/momentum/class_2/assigned/samples": 2600.0, "metrics/particle/momentum/class_2/efficiency": 0.46401384472846985, "metrics/particle/momentum/class_2/event/energy_bias": -5.883819103240967, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.882307052612305, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.626087188720703, "metrics/particle/momentum/class_2/event/px_bias": -0.40827658772468567, "metrics/particle/momentum/class_2/event/py_bias": 0.1470438838005066, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.319059371948242, "metrics/particle/momentum/class_2/event/pz_bias": -1.240561842918396, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.100221633911133, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021357469260692596, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.133397176861763, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1314917951822281, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.029220037162303925, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1730843037366867, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000748876656871289, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17086344957351685, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.004714920185506344, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6307846307754517, "metrics/particle/momentum/class_3/accepted/samples": 12018.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2669658660888672, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3483366966247559, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2353888750076294, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1071189641952515, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1430034190416336, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.147716522216797, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008054607547819614, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.46841299533843994, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.031064994633197784, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.839524030685425, "metrics/particle/momentum/class_3/assigned/samples": 14650.0, "metrics/particle/momentum/class_3/efficiency": 0.7912304997444153, "metrics/particle/momentum/class_3/event/energy_bias": 4.546390056610107, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.792610168457031, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.82689094543457, "metrics/particle/momentum/class_3/event/px_bias": -0.3597770035266876, "metrics/particle/momentum/class_3/event/py_bias": -0.21162575483322144, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.2197980880737305, "metrics/particle/momentum/class_3/event/pz_bias": 1.3192040920257568, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.815853118896484, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011716870591044426, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1615014523267746, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10627923160791397, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05475820600986481, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1593766063451767, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00671140942722559, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.42216429114341736, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.14653967320919037, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4687130451202393, "metrics/particle/momentum/class_4/accepted/samples": 298.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2143671065568924, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.8828206062316895, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6491354703903198, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.594378113746643, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12994350492954254, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.585790634155273, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.581081211566925, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.11135388165712357, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1374919414520264, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8418079018592834, "metrics/particle/momentum/class_4/event/energy_bias": -1.4131759405136108, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8686686754226685, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4397547245025635, "metrics/particle/momentum/class_4/event/px_bias": -0.15498226881027222, "metrics/particle/momentum/class_4/event/py_bias": 0.2092454433441162, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9621431827545166, "metrics/particle/momentum/class_4/event/pz_bias": 0.5296424031257629, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.343794584274292, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005356852430850267, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02174552157521248, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021061299368739128, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004341436084359884, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0537838451564312, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14481529593467712, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04509935900568962, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5920665264129639, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0141056589782238, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.040365301072597504, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03813944011926651, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.003288162872195244, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12459801882505417, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.1740068644285202, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.04504719749093056, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.6361672282218933, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.06756482273340225, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8179776668548584, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6814662218093872, "metrics/particle/momentum/class_5/event/px_bias": 0.09012064337730408, "metrics/particle/momentum/class_5/event/py_bias": -0.03635597229003906, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3376281261444092, "metrics/particle/momentum/class_5/event/pz_bias": 0.12650778889656067, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5021954774856567, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.9586260318756104, "step_valid_time": 16487.924366235733, "step_total_time": 16489.882992506027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030132914594936, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06918684779714135, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9632425336396455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.96419185282523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9637169594483664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03580814717477004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9615637319316689, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9610901329830898, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9632425336396455, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000147761865852, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07715506676533013, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2272, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2169, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568661971830986, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556043956043956, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043133802816901406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9521510096575944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9546654929577465, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534065934065934, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543459174714662, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035096209657337, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0776255808553965, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06069802731411229, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9393019726858877} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_38000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..103f261d366840e9a88c2edd5adec417923f9760 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.7999242137173173, "train": {"Classification_binary": 2.4623641967773438, "Classification": 0.015996932983398438, "Regression_pt": 0.1201862022280693, "Regression_eta": 0.006669863127171993, "Regression_sin_phi": 0.006277835927903652, "Regression_cos_phi": 0.005757822189480066, "Regression_energy": 0.12011472135782242, "Total": 7.091815948486328, "Total_unweighted": 2.7373675745911896}, "train_group": {}, "valid": {"Classification_binary": 2.373412768046061, "Classification": 0.0172534075876077, "Regression_pt": 0.12195125222206116, "Regression_eta": 0.010873729983965555, "Regression_sin_phi": 0.006417473157246907, "Regression_cos_phi": 0.006828784942626953, "Regression_energy": 0.12661104400952658, "Total": 7.444394429524739, "Total_unweighted": 2.6633484599490957, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017238110303878784, "diagnostic/loss/cld_pf/Classification_binary": 2.3719141483306885, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006670478265732527, "diagnostic/loss/cld_pf/Regression_energy": 0.12603232264518738, "diagnostic/loss/cld_pf/Regression_eta": 0.010325837880373001, "diagnostic/loss/cld_pf/Regression_pt": 0.12209218740463257, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006010062992572784, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08573754876852036, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1832992434501648, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05861828103661537, "diagnostic/regression/cld_pf/energy_residual_rms": 0.40855714678764343, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03694148734211922, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17914582788944244, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06410862505435944, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40220510959625244, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9479473829269409, "metrics/particle/class_1/pid_accuracy": 0.9870607852935791, "metrics/particle/class_2/efficiency": 0.46769967675209045, "metrics/particle/class_2/pid_accuracy": 0.7834348082542419, "metrics/particle/class_3/efficiency": 0.7928533554077148, "metrics/particle/class_3/pid_accuracy": 0.9867509007453918, "metrics/particle/class_4/efficiency": 0.8314833641052246, "metrics/particle/class_4/pid_accuracy": 0.3956310749053955, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.6932271122932434, "metrics/particle/count/bias_mean": -0.20980392396450043, "metrics/particle/count/mae": 3.0163397789001465, "metrics/particle/count/prediction_mean": 70.35555267333984, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04596244916319847, "metrics/particle/event/energy_response_mean": 0.9922783970832825, "metrics/particle/event/met_abs_error": 4.409934043884277, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03677476942539215, "metrics/particle/event/scalar_pt_response_mean": 1.003324031829834, "metrics/particle/event/vector_pt_closure": 0.04097502678632736, "metrics/particle/matched/delta_eta_abs_mean": 0.007883374579250813, "metrics/particle/matched/delta_phi_abs_mean": 0.008426392450928688, "metrics/particle/matched/delta_r_mean": 0.01300870906561613, "metrics/particle/matched/energy_relative_abs_mean": 0.08571675419807434, "metrics/particle/matched/pid_accuracy": 0.9707708954811096, "metrics/particle/matched/pt_relative_abs_mean": 0.08142464607954025, "metrics/particle/matching/duplicate_fraction": 0.028296979144215584, "metrics/particle/matching/efficiency": 0.8425971269607544, "metrics/particle/matching/f1": 0.843851625919342, "metrics/particle/matching/purity": 0.8451098203659058, "metrics/particle/matching/target_coverage_dr0p05": 0.829935610294342, "metrics/particle/matching/target_coverage_dr0p10": 0.8753485083580017, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7414439916610718, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8762747049331665, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015147517435252666, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08248685300350189, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08050492405891418, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01909610815346241, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.133040651679039, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004646729212254286, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.35224130749702454, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0010763159953057766, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5012253522872925, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8393.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23417766392230988, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8011435270309448, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7871549129486084, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6861554384231567, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10905767977237701, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.631440162658691, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013606823980808258, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6647025942802429, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.005719342269003391, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5297908782958984, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9848.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8279569745063782, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3296576738357544, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.861333847045898, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5019917488098145, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.3220091462135315, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.010411988943815231, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.10123348236084, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7182419300079346, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.623392105102539, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024389902129769325, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09000781923532486, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09014051407575607, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01947290264070034, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1417199969291687, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02738892287015915, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9831736087799072, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.11466176062822342, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2064645290374756, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1643.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7342830300331116, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5678499937057495, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17450.015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17449.861328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24578596651554108, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 881206.625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07056056708097458, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.566258192062378, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.02944827266037464, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.659355640411377, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2551.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5976718664169312, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.21297287940979, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.8504767417907715, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8043212294578552, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07487255334854126, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06635133922100067, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.733675956726074, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.4661515951156616, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.18825626373291, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010928825475275517, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08785989135503769, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08217420428991318, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015492133796215057, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13618817925453186, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00034921427140943706, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10407647490501404, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.000564460176974535, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4741854667663574, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20045.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13220545649528503, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6035836935043335, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.48449116945266724, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.37940916419029236, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0853663980960846, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.3045735359191895, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002488557016476989, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22384701669216156, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0009671711013652384, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1021733283996582, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22503.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8767440915107727, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.3256363272666931, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.585686206817627, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.8141937255859375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.14786669611930847, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2614150941371918, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.3172597885131836, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.08245600759983063, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 3.8012521266937256, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012841129675507545, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08647805452346802, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08214356005191803, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016715116798877716, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13562723994255066, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003025165293365717, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2213333398103714, "metrics/particle/momentum/all/accepted/pz_bias": -0.00693917740136385, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1576712131500244, "metrics/particle/momentum/all/accepted/samples": 30081.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2049841731786728, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7298060655593872, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1275.9625244140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1275.8551025390625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10377629101276398, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 238236.234375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010601111687719822, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5194472074508667, "metrics/particle/momentum/all/assigned/pz_bias": -0.004389744717627764, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6223878860473633, "metrics/particle/momentum/all/assigned/samples": 34902.0, "metrics/particle/momentum/all/efficiency": 0.8414500951766968, "metrics/particle/momentum/all/event/energy_bias": -2.216994047164917, "metrics/particle/momentum/all/event/p3_error_mean": 12.189643859863281, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.470643520355225, "metrics/particle/momentum/all/event/px_bias": -0.5447484254837036, "metrics/particle/momentum/all/event/py_bias": -0.20547571778297424, "metrics/particle/momentum/all/event/pz_abs_mean": 8.81636905670166, "metrics/particle/momentum/all/event/pz_bias": -0.3345463275909424, "metrics/particle/momentum/all/event/pz_residual_rms": 15.869938850402832, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007552261929959059, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04391082748770714, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.036741968244314194, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007719792425632477, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08345720171928406, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001604443066753447, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13542604446411133, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007268325891345739, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.884333610534668, "metrics/particle/momentum/class_1/accepted/samples": 16205.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06828062981367111, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2315746694803238, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.26133835315704346, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21360133588314056, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03892390429973602, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.552547454833984, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0044351075775921345, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.25004246830940247, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.017043793573975563, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.5213476419448853, "metrics/particle/momentum/class_1/assigned/samples": 17136.0, "metrics/particle/momentum/class_1/efficiency": 0.9452837705612183, "metrics/particle/momentum/class_1/event/energy_bias": 1.7550461292266846, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.231874465942383, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.727245807647705, "metrics/particle/momentum/class_1/event/px_bias": 0.10075712203979492, "metrics/particle/momentum/class_1/event/py_bias": -0.009814411401748657, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.930689811706543, "metrics/particle/momentum/class_1/event/pz_bias": -0.8510013818740845, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.313820838928223, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032588280737400055, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18735741078853607, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1885329931974411, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03953296318650246, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22882770001888275, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0395817756652832, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7053585052490234, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10016001015901566, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.015557289123535, "metrics/particle/momentum/class_2/accepted/samples": 1339.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8149203658103943, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6638182997703552, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17112.576171875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17112.126953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3498654365539551, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 872695.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0703575536608696, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7837255001068115, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06278353929519653, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.45451021194458, "metrics/particle/momentum/class_2/assigned/samples": 2601.0, "metrics/particle/momentum/class_2/efficiency": 0.4633218050003052, "metrics/particle/momentum/class_2/event/energy_bias": -7.069437026977539, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.317468643188477, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.867019176483154, "metrics/particle/momentum/class_2/event/px_bias": -0.1080927848815918, "metrics/particle/momentum/class_2/event/py_bias": 0.3124622106552124, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.637508392333984, "metrics/particle/momentum/class_2/event/pz_bias": -0.9320953488349915, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.201704025268555, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01788562349975109, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13143962621688843, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13140904903411865, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03421860188245773, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17304711043834686, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008276775479316711, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16792239248752594, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0005546185420826077, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6675059199333191, "metrics/particle/momentum/class_3/accepted/samples": 12082.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25699758529663086, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2926467657089233, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2860430479049683, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.16717529296875, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13581089675426483, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.22417163848877, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0072414265014231205, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.43423008918762207, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019285988062620163, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4868829250335693, "metrics/particle/momentum/class_3/assigned/samples": 14638.0, "metrics/particle/momentum/class_3/efficiency": 0.7954440712928772, "metrics/particle/momentum/class_3/event/energy_bias": 4.305165767669678, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.504048347473145, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.666258811950684, "metrics/particle/momentum/class_3/event/px_bias": -0.4221835136413574, "metrics/particle/momentum/class_3/event/py_bias": -0.5709511041641235, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.999542713165283, "metrics/particle/momentum/class_3/event/pz_bias": 0.9101825952529907, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.240983009338379, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011029019951820374, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16134455800056458, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10823118686676025, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.051624298095703125, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16294553875923157, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0069444444961845875, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.45559629797935486, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10803565382957458, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4789793491363525, "metrics/particle/momentum/class_4/accepted/samples": 288.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2790749669075012, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.3924427032470703, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4613677263259888, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4036402702331543, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1497175097465515, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.09345817565918, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.599647581577301, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08154641091823578, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9247008562088013, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8135592937469482, "metrics/particle/momentum/class_4/event/energy_bias": -1.29862642288208, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.760022521018982, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2976268529891968, "metrics/particle/momentum/class_4/event/px_bias": -0.1982346773147583, "metrics/particle/momentum/class_4/event/py_bias": 0.0836905837059021, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9520489573478699, "metrics/particle/momentum/class_4/event/pz_bias": 0.44539737701416016, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.401364326477051, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005888807121664286, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026222383603453636, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025488128885626793, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.014047387056052685, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07358255982398987, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11870706081390381, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.032005514949560165, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.47285696864128113, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.022942688316106796, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.047089993953704834, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04428247734904289, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.026959476992487907, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13918447494506836, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20817622542381287, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05216280370950699, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9500116109848022, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.09085755795240402, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7925523519515991, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6552561521530151, "metrics/particle/momentum/class_5/event/px_bias": 0.08300544321537018, "metrics/particle/momentum/class_5/event/py_bias": -0.02086298167705536, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.34237146377563477, "metrics/particle/momentum/class_5/event/pz_bias": 0.09297040849924088, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.502859354019165, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.8753647804260254, "step_valid_time": 16936.781185150146, "step_total_time": 16938.65655016899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035738408768777, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06862249186214886, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2951, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2943, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9684936002625534, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.968811556139199, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9686525521089775, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03118844386080105, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9658680669510994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9661851608667105, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.96602658788774, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9684936002625534, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026296757807156, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07209979400229138, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2179, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.956540825285338, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9582233948988567, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573813708260105, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04177660510114336, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9564643799472295, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556239015817223, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.956540825285338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054942220929954, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07533666662861915, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1320, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9371212121212121, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9378316906747536, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06287878787878788, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936267071320182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348484848484848, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.935557240333586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385432473444613} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_39000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..9c85d483efd6ebbe731c6f249006ba85dd4c5874 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.8472906403940887, "train": {"Classification_binary": 2.2705283164978027, "Classification": 0.018107548356056213, "Regression_pt": 0.10916440188884735, "Regression_eta": 0.009686441160738468, "Regression_sin_phi": 0.005540946498513222, "Regression_cos_phi": 0.005965679883956909, "Regression_energy": 0.10989049822092056, "Total": 6.87007999420166, "Total_unweighted": 2.5288838325068355}, "train_group": {}, "valid": {"Classification_binary": 2.377226193745931, "Classification": 0.017216873665650684, "Regression_pt": 0.12190528710683186, "Regression_eta": 0.011091521630684534, "Regression_sin_phi": 0.006411411489049594, "Regression_cos_phi": 0.006762318934003512, "Regression_energy": 0.12667805949846903, "Total": 7.452340443929036, "Total_unweighted": 2.6672916660706205, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.017164651304483414, "diagnostic/loss/cld_pf/Classification_binary": 2.3781611919403076, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006622682325541973, "diagnostic/loss/cld_pf/Regression_energy": 0.1254916787147522, "diagnostic/loss/cld_pf/Regression_eta": 0.010385898873209953, "diagnostic/loss/cld_pf/Regression_pt": 0.12137918174266815, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006052932236343622, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09080251306295395, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18481554090976715, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06368323415517807, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4116564691066742, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03864385187625885, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.180137038230896, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06581098586320877, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40355369448661804, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9466058611869812, "metrics/particle/class_1/pid_accuracy": 0.9866577386856079, "metrics/particle/class_2/efficiency": 0.4614332914352417, "metrics/particle/class_2/pid_accuracy": 0.7834568023681641, "metrics/particle/class_3/efficiency": 0.7888097763061523, "metrics/particle/class_3/pid_accuracy": 0.9870730638504028, "metrics/particle/class_4/efficiency": 0.8143289685249329, "metrics/particle/class_4/pid_accuracy": 0.41263940930366516, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6739562749862671, "metrics/particle/count/bias_mean": -0.4470588266849518, "metrics/particle/count/mae": 3.05751633644104, "metrics/particle/count/prediction_mean": 70.11830139160156, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04557667672634125, "metrics/particle/event/energy_response_mean": 1.0001628398895264, "metrics/particle/event/met_abs_error": 4.781768321990967, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037266042083501816, "metrics/particle/event/scalar_pt_response_mean": 1.007460594177246, "metrics/particle/event/vector_pt_closure": 0.041401155292987823, "metrics/particle/matched/delta_eta_abs_mean": 0.00867390912026167, "metrics/particle/matched/delta_phi_abs_mean": 0.008374291472136974, "metrics/particle/matched/delta_r_mean": 0.01359020359814167, "metrics/particle/matched/energy_relative_abs_mean": 0.08619634062051773, "metrics/particle/matched/pid_accuracy": 0.9708977937698364, "metrics/particle/matched/pt_relative_abs_mean": 0.0812097042798996, "metrics/particle/matching/duplicate_fraction": 0.028467295691370964, "metrics/particle/matching/efficiency": 0.8395869135856628, "metrics/particle/matching/f1": 0.8422549366950989, "metrics/particle/matching/purity": 0.8449399471282959, "metrics/particle/matching/target_coverage_dr0p05": 0.8263233304023743, "metrics/particle/matching/target_coverage_dr0p10": 0.8722641468048096, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7397027015686035, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.873931348323822, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015788795426487923, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08218405395746231, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07944919914007187, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018787963315844536, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13136234879493713, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004557994659990072, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3484379053115845, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.008246175944805145, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4780961275100708, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8337.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23184752464294434, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8178236484527588, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.774530291557312, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6760062575340271, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10884631425142288, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.56679105758667, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01192417461425066, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.64857417345047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02545802667737007, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.3891303539276123, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9812.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8224326968193054, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.18013662099838257, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.399232387542725, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7896885871887207, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.025676652789115906, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1422528624534607, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.555712699890137, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.46657076478004456, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.517974853515625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024434572085738182, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09197437018156052, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0920829325914383, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.022258728742599487, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1439332365989685, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02083333395421505, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9409661889076233, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19508099555969238, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.090839385986328, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1632.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7273766398429871, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5895721912384033, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 34852.39453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 34852.234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25262749195098877, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1766352.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07707279175519943, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.672663450241089, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.005800789222121239, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.437664985656738, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2569.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5936704277992249, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.6711057424545288, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.079601764678955, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8110503554344177, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10757945477962494, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07007003575563431, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.970198631286621, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.4192776679992676, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.111091613769531, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0115536293014884, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08845558017492294, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08208926767110825, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016392648220062256, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1360902637243271, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000249613105552271, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10495995730161667, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.009615635499358177, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4592181444168091, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20031.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13334012031555176, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6114532351493835, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4452585279941559, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3401503264904022, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08395621925592422, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.721318244934082, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024915465619415045, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2315756231546402, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.015166557393968105, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1779961585998535, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22476.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8761317133903503, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.4623294174671173, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.121310710906982, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.145441055297852, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.23456160724163055, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.17165492475032806, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.832524299621582, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.043258048593997955, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.462452411651611, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01343130599707365, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08690415322780609, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0818992480635643, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017377421259880066, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13523545861244202, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025666665751487017, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21810121834278107, "metrics/particle/momentum/all/accepted/pz_bias": -0.0019004347268491983, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1259002685546875, "metrics/particle/momentum/all/accepted/samples": 30000.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.204850435256958, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7416335344314575, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2569.165771484375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2569.058349609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10339386761188507, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 479528.59375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010643485933542252, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5288687348365784, "metrics/particle/momentum/all/assigned/pz_bias": 0.01651821844279766, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5648763179779053, "metrics/particle/momentum/all/assigned/samples": 34857.0, "metrics/particle/momentum/all/efficiency": 0.8391842842102051, "metrics/particle/momentum/all/event/energy_bias": -0.02863970585167408, "metrics/particle/momentum/all/event/p3_error_mean": 12.671052932739258, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.634227752685547, "metrics/particle/momentum/all/event/px_bias": 0.15265882015228271, "metrics/particle/momentum/all/event/py_bias": -0.24383777379989624, "metrics/particle/momentum/all/event/pz_abs_mean": 9.28995418548584, "metrics/particle/momentum/all/event/pz_bias": 0.9959650039672852, "metrics/particle/momentum/all/event/pz_residual_rms": 15.824048042297363, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007809005677700043, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.044479887932538986, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03628510609269142, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01079263724386692, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08190570771694183, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.000925697386264801, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12935741245746613, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.002632411429658532, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7895331978797913, "metrics/particle/momentum/class_1/accepted/samples": 16204.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06680302321910858, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2101602554321289, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.24911682307720184, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20589865744113922, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03955890238285065, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.342434406280518, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00361748063005507, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2473069131374359, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0028520531486719847, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.506894826889038, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9452254772186279, "metrics/particle/momentum/class_1/event/energy_bias": 3.5592973232269287, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.280759811401367, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.849733352661133, "metrics/particle/momentum/class_1/event/px_bias": 0.45639270544052124, "metrics/particle/momentum/class_1/event/py_bias": -0.3221971094608307, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.969606399536133, "metrics/particle/momentum/class_1/event/pz_bias": 0.01077774167060852, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.32601261138916, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032664619386196136, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1880916804075241, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18952052295207977, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0435844361782074, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22944676876068115, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.040273554623126984, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7349860668182373, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.16474922001361847, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.026210784912109, "metrics/particle/momentum/class_2/accepted/samples": 1316.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7945880889892578, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7084582448005676, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 34355.5546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 34355.10546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35264772176742554, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1753768.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0752110481262207, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8556809425354004, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.02528742328286171, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.335445404052734, "metrics/particle/momentum/class_2/assigned/samples": 2606.0, "metrics/particle/momentum/class_2/efficiency": 0.45536333322525024, "metrics/particle/momentum/class_2/event/energy_bias": -7.154915809631348, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.764969825744629, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.418807029724121, "metrics/particle/momentum/class_2/event/px_bias": -0.13483291864395142, "metrics/particle/momentum/class_2/event/py_bias": -0.02712930738925934, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.393046379089355, "metrics/particle/momentum/class_2/event/pz_bias": -0.21585345268249512, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.094615936279297, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019056960940361023, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13214509189128876, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13174039125442505, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.031901393085718155, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17355380952358246, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006651700241491199, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16889633238315582, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0072240461595356464, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7172291278839111, "metrics/particle/momentum/class_3/accepted/samples": 12027.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2627701759338379, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.318007469177246, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2236775159835815, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1006861925125122, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13342474400997162, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.093554496765137, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007404868025332689, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.44429662823677063, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02774098329246044, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3856756687164307, "metrics/particle/momentum/class_3/assigned/samples": 14585.0, "metrics/particle/momentum/class_3/efficiency": 0.7918230295181274, "metrics/particle/momentum/class_3/event/energy_bias": 4.728213787078857, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.604931831359863, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.739195346832275, "metrics/particle/momentum/class_3/event/px_bias": -0.0709753930568695, "metrics/particle/momentum/class_3/event/py_bias": -0.03244711086153984, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.079893112182617, "metrics/particle/momentum/class_3/event/pz_bias": 0.7899380922317505, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.911943435668945, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011049134656786919, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16038008034229279, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11099463701248169, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06527279317378998, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16577470302581787, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003508772002533078, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.38238951563835144, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06014255806803703, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1388493776321411, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24920803308486938, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.314450979232788, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9925049543380737, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.944502353668213, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16384181380271912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.281290054321289, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6958226561546326, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12962545454502106, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.779874086380005, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.805084764957428, "metrics/particle/momentum/class_4/event/energy_bias": -1.1753041744232178, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6858038902282715, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.286028265953064, "metrics/particle/momentum/class_4/event/px_bias": -0.20987048745155334, "metrics/particle/momentum/class_4/event/py_bias": 0.15904715657234192, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8769071698188782, "metrics/particle/momentum/class_4/event/pz_bias": 0.31760373711586, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.139673948287964, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006359035614877939, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022775914520025253, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021007642149925232, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009018274024128914, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.058513619005680084, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1392274647951126, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0780780166387558, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4829222559928894, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02378489077091217, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0375007800757885, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.033984895795583725, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01566607505083084, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10677013546228409, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.16121619939804077, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06072534993290901, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.5220580697059631, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.014069341123104095, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.732327938079834, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6128073334693909, "metrics/particle/momentum/class_5/event/px_bias": 0.11194488406181335, "metrics/particle/momentum/class_5/event/py_bias": -0.021111363545060158, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3036462068557739, "metrics/particle/momentum/class_5/event/pz_bias": 0.09349895268678665, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.3999178409576416, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.774440050125122, "step_valid_time": 17380.721228837967, "step_total_time": 17382.49566912651, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023033410723383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06794173512957968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2925, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9632425336396455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9657782165185916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9645087085113375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.034221783481408356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9599606170003282, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9624876604146101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9612224778179428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9632425336396455, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0051438531325776, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07395560747596919, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2281, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2177, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2170, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9544059622972381, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9550339986839219, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04559403770276195, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9525899912203687, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9513371328364753, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9519631498135556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9556628621597892, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008618709669767, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07485760891358006, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412213740458015, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9383561643835616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05877862595419847, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9339908952959028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9396946564885497, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.936834094368341, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9355083459787557} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_4000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..73eb0ed7edc1f39851f65b20cf3f848b77930a7a --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.189465706707086, "train": {"Classification_binary": 3.5207974910736084, "Classification": 0.02578066475689411, "Regression_pt": 0.2075847089290619, "Regression_eta": 0.008958208374679089, "Regression_sin_phi": 0.007236429490149021, "Regression_cos_phi": 0.007040968164801598, "Regression_energy": 0.2083570659160614, "Total": 10.882528305053711, "Total_unweighted": 3.9857555367052555}, "train_group": {}, "valid": {"Classification_binary": 3.4988174438476562, "Classification": 0.027143952747186024, "Regression_pt": 0.19093205531438193, "Regression_eta": 0.014521756519873938, "Regression_sin_phi": 0.008152871082226435, "Regression_cos_phi": 0.00860905833542347, "Regression_energy": 0.19632802406946817, "Total": 11.034105936686197, "Total_unweighted": 3.9445051619162164, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.027228448539972305, "diagnostic/loss/cld_pf/Classification_binary": 3.4755454063415527, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.008240780793130398, "diagnostic/loss/cld_pf/Regression_energy": 0.1951630562543869, "diagnostic/loss/cld_pf/Regression_eta": 0.013398958370089531, "diagnostic/loss/cld_pf/Regression_pt": 0.1909199357032776, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007670182269066572, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09818953275680542, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.22764746844768524, "diagnostic/regression/cld_pf/energy_residual_mean": 0.07107025384902954, "diagnostic/regression/cld_pf/energy_residual_rms": 0.49054810404777527, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.053233176469802856, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.2236870676279068, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08040031790733337, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4852954149246216, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9119935631752014, "metrics/particle/class_1/pid_accuracy": 0.9812760353088379, "metrics/particle/class_2/efficiency": 0.285063236951828, "metrics/particle/class_2/pid_accuracy": 0.47362110018730164, "metrics/particle/class_3/efficiency": 0.7000483274459839, "metrics/particle/class_3/pid_accuracy": 0.9778998494148254, "metrics/particle/class_4/efficiency": 0.8567103743553162, "metrics/particle/class_4/pid_accuracy": 0.19316843152046204, "metrics/particle/class_5/efficiency": 0.9533980488777161, "metrics/particle/class_5/pid_accuracy": 0.42566192150115967, "metrics/particle/count/bias_mean": -2.9281046390533447, "metrics/particle/count/mae": 4.946405410766602, "metrics/particle/count/prediction_mean": 67.63725280761719, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.07533791661262512, "metrics/particle/event/energy_response_mean": 0.9813439846038818, "metrics/particle/event/met_abs_error": 7.79752779006958, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06318332254886627, "metrics/particle/event/scalar_pt_response_mean": 0.9838282465934753, "metrics/particle/event/vector_pt_closure": 0.06702109426259995, "metrics/particle/matched/delta_eta_abs_mean": 0.015321935527026653, "metrics/particle/matched/delta_phi_abs_mean": 0.013408285565674305, "metrics/particle/matched/delta_r_mean": 0.022693365812301636, "metrics/particle/matched/energy_relative_abs_mean": 0.09277070313692093, "metrics/particle/matched/pid_accuracy": 0.9534238576889038, "metrics/particle/matched/pt_relative_abs_mean": 0.08811008930206299, "metrics/particle/matching/duplicate_fraction": 0.040749866515398026, "metrics/particle/matching/efficiency": 0.7713888883590698, "metrics/particle/matching/f1": 0.7877323031425476, "metrics/particle/matching/purity": 0.804783284664154, "metrics/particle/matching/target_coverage_dr0p05": 0.6998564600944519, "metrics/particle/matching/target_coverage_dr0p10": 0.8070022463798523, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6711156368255615, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8141804933547974, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026863237842917442, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0903322696685791, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08748879283666611, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019662750884890556, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14145046472549438, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006301693618297577, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.42363908886909485, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.015834707766771317, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.746103286743164, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7617.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.342791885137558, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4161759614944458, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4157304763793945, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2950832843780518, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15164446830749512, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.607878684997559, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022689426317811012, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9993255138397217, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08718395978212357, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5477371215820312, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9608.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7514057159423828, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.9717769622802734, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.3980073928833, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.654516220092773, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3397623300552368, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7862532734870911, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.917187690734863, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.3109421730041504, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.987756729125977, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0321415476500988, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10228821635246277, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10294175148010254, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.028224334120750427, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1565648317337036, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0199115052819252, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0429781675338745, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15602785348892212, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.279071092605591, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1356.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8433249592781067, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.2366466522216797, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 15038.302734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 15038.1484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3079800605773926, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 737430.5625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08728179335594177, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2481703758239746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1275140345096588, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.6979398727417, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2406.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.49327027797698975, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.1391574144363403, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.951220989227295, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.385546088218689, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.012591172009706497, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04170764237642288, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.7273173332214355, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.9735397696495056, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.619749069213867, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02047915942966938, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09558712691068649, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09014405310153961, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012132573872804642, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14634571969509125, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006944073247723281, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13048602640628815, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.021164122968912125, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5569920539855957, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18721.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20452144742012024, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.075080156326294, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7721070051193237, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6434376239776611, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1122887134552002, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.426470756530762, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0066697122529149055, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.38901257514953613, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.06066210940480232, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.213383674621582, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21890.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8188339471817017, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -6.051501274108887, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.618080139160156, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.955570220947266, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9976862668991089, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.8338103294372559, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.632357120513916, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.881737470626831, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.839970588684082, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0228060781955719, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09446992725133896, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09004037082195282, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014991596341133118, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14553743600845337, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003177583683282137, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2557942569255829, "metrics/particle/momentum/all/accepted/pz_bias": 0.01102234236896038, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.254894733428955, "metrics/particle/momentum/all/accepted/samples": 27694.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.28903841972351074, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2541732788085938, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1068.093994140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1067.9659423828125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13732892274856567, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 196446.125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01693015545606613, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7648684978485107, "metrics/particle/momentum/all/assigned/pz_bias": 0.05482415482401848, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4783213138580322, "metrics/particle/momentum/all/assigned/samples": 33904.0, "metrics/particle/momentum/all/efficiency": 0.7746790051460266, "metrics/particle/momentum/all/event/energy_bias": -2.9405670166015625, "metrics/particle/momentum/all/event/p3_error_mean": 21.04185676574707, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.278139114379883, "metrics/particle/momentum/all/event/px_bias": -0.6705151200294495, "metrics/particle/momentum/all/event/py_bias": 2.661771297454834, "metrics/particle/momentum/all/event/pz_abs_mean": 14.881216049194336, "metrics/particle/momentum/all/event/pz_bias": 4.166219711303711, "metrics/particle/momentum/all/event/pz_residual_rms": 22.76987648010254, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012530158273875713, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05361619591712952, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04732096940279007, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006454930175095797, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09920445084571838, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019828579388558865, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16706445813179016, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.019541621208190918, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9159864187240601, "metrics/particle/momentum/class_1/accepted/samples": 15634.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11291272938251495, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33002933859825134, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.39310991764068604, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.31916821002960205, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06517679989337921, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.479941368103027, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008402381092309952, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.41426563262939453, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.042384110391139984, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4774892330169678, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9119757413864136, "metrics/particle/momentum/class_1/event/energy_bias": 1.8183872699737549, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.702790260314941, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.758977890014648, "metrics/particle/momentum/class_1/event/px_bias": -0.25160157680511475, "metrics/particle/momentum/class_1/event/py_bias": 0.455383837223053, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.818183898925781, "metrics/particle/momentum/class_1/event/pz_bias": 1.154038667678833, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.843931198120117, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04811239615082741, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21875211596488953, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22012506425380707, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06363267451524734, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26025110483169556, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.05092592537403107, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.2715260982513428, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.19560889899730682, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.950804710388184, "metrics/particle/momentum/class_2/accepted/samples": 864.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1028714179992676, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.026929497718811, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 16420.826171875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 16420.296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4421243667602539, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 770658.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.10304130613803864, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.9022700786590576, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.01987435109913349, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.924839973449707, "metrics/particle/momentum/class_2/assigned/samples": 2203.0, "metrics/particle/momentum/class_2/efficiency": 0.29896193742752075, "metrics/particle/momentum/class_2/event/energy_bias": -11.898198127746582, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.775352478027344, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.35651969909668, "metrics/particle/momentum/class_2/event/px_bias": 0.06006607413291931, "metrics/particle/momentum/class_2/event/py_bias": 1.342804193496704, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.119752883911133, "metrics/particle/momentum/class_2/event/pz_bias": 1.0100080966949463, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.609766006469727, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03609032556414604, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14318326115608215, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1422429382801056, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.033433523029088974, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18451841175556183, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010253542568534613, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21597954630851746, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.009231936186552048, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8564698100090027, "metrics/particle/momentum/class_3/accepted/samples": 10728.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3813179135322571, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4085521697998047, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.1467249393463135, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.013241767883301, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17924058437347412, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.770222663879395, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.014248059131205082, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7069675922393799, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07089117914438248, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4066195487976074, "metrics/particle/momentum/class_3/assigned/samples": 14037.0, "metrics/particle/momentum/class_3/efficiency": 0.7063006162643433, "metrics/particle/momentum/class_3/event/energy_bias": 10.025375366210938, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.779115676879883, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.133943557739258, "metrics/particle/momentum/class_3/event/px_bias": -0.7302522659301758, "metrics/particle/momentum/class_3/event/py_bias": 0.7164671421051025, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.058038711547852, "metrics/particle/momentum/class_3/event/pz_bias": 1.7209678888320923, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.052001953125, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01739581488072872, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16133280098438263, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11167054623365402, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03547066077589989, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1665154993534088, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006557377055287361, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.52911376953125, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17777614295482635, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.146688938140869, "metrics/particle/momentum/class_4/accepted/samples": 305.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.19575108587741852, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.21501088142395, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.969067096710205, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8909590244293213, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.11331444978713989, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.610803604125977, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008498583920300007, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7195892930030823, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2238970249891281, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.187548875808716, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8615819215774536, "metrics/particle/momentum/class_4/event/energy_bias": -2.1946587562561035, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0985991954803467, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1604857444763184, "metrics/particle/momentum/class_4/event/px_bias": 0.16153229773044586, "metrics/particle/momentum/class_4/event/py_bias": 0.2353658229112625, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8588755130767822, "metrics/particle/momentum/class_4/event/pz_bias": 0.18280597031116486, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.216873645782471, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010081127285957336, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022921904921531677, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021669737994670868, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0015575055731460452, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05513083562254906, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19063439965248108, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.09498897194862366, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5077764391899109, "metrics/particle/momentum/class_5/accepted/samples": 163.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07615626603364944, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07154359668493271, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.07062649726867676, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.011722533963620663, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05202312022447586, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.23249641060829163, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.33524367213249207, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.08358929306268692, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9638081789016724, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9421965479850769, "metrics/particle/momentum/class_5/event/energy_bias": -0.691471517086029, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.2367277145385742, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9822841286659241, "metrics/particle/momentum/class_5/event/px_bias": 0.0897403284907341, "metrics/particle/momentum/class_5/event/py_bias": -0.08824983984231949, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6151939034461975, "metrics/particle/momentum/class_5/event/pz_bias": 0.09839875996112823, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.3230693340301514, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.477576732635498, "step_valid_time": 1783.5767621994019, "step_total_time": 1785.054339170456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.987216856740577, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1331667462097086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3008, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2828, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2809, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9281260255989497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9401595744680851, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9341040462427745, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0598404255319149, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9218903839842468, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.933843085106383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9278282411230389, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9281260255989497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9938694534634227, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14098280798711071, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2252, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2104, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2085, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9236172080772608, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9342806394316163, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9289183222958057, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06571936056838366, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9152765583845478, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9258436944937833, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9205298013245033, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9236172080772608, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998507099332342, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14418501465408928, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1312, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1177, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8990895295902883, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9032012195121951, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9011406844106464, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09679878048780488, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8930197268588771, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8971036585365854, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8950570342205323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8990895295902883} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_40000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..0773f2ae95c7dfe4c978106980918dcf9ba95d40 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.8946570670708602, "train": {"Classification_binary": 2.3908021450042725, "Classification": 0.016666660085320473, "Regression_pt": 0.133054718375206, "Regression_eta": 0.008159304969012737, "Regression_sin_phi": 0.004949051421135664, "Regression_cos_phi": 0.0044371518306434155, "Regression_energy": 0.13038475811481476, "Total": 7.214469909667969, "Total_unweighted": 2.6884537898004055}, "train_group": {}, "valid": {"Classification_binary": 2.360752900441488, "Classification": 0.01718027393023173, "Regression_pt": 0.12102895975112915, "Regression_eta": 0.010915353894233704, "Regression_sin_phi": 0.006384892389178276, "Regression_cos_phi": 0.006744823108116786, "Regression_energy": 0.12575913468996683, "Total": 7.4020233154296875, "Total_unweighted": 2.6487663382043443, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01701386086642742, "diagnostic/loss/cld_pf/Classification_binary": 2.3608405590057373, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0065232133492827415, "diagnostic/loss/cld_pf/Regression_energy": 0.12397251278162003, "diagnostic/loss/cld_pf/Regression_eta": 0.010274086147546768, "diagnostic/loss/cld_pf/Regression_pt": 0.1200251504778862, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00598911801353097, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08746528625488281, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18302036821842194, "diagnostic/regression/cld_pf/energy_residual_mean": 0.060346003621816635, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4086472690105438, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03397031128406525, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17777632176876068, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06113744527101517, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4007006883621216, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9488481879234314, "metrics/particle/class_1/pid_accuracy": 0.9870730638504028, "metrics/particle/class_2/efficiency": 0.47271278500556946, "metrics/particle/class_2/pid_accuracy": 0.7871776223182678, "metrics/particle/class_3/efficiency": 0.7905678749084473, "metrics/particle/class_3/pid_accuracy": 0.9863512516021729, "metrics/particle/class_4/efficiency": 0.8213925361633301, "metrics/particle/class_4/pid_accuracy": 0.43488943576812744, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6679920554161072, "metrics/particle/count/bias_mean": -0.4614379107952118, "metrics/particle/count/mae": 3.0967321395874023, "metrics/particle/count/prediction_mean": 70.10391998291016, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.045377638190984726, "metrics/particle/event/energy_response_mean": 0.9951627254486084, "metrics/particle/event/met_abs_error": 4.37168025970459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03629746660590172, "metrics/particle/event/scalar_pt_response_mean": 1.001426100730896, "metrics/particle/event/vector_pt_closure": 0.04006332531571388, "metrics/particle/matched/delta_eta_abs_mean": 0.00787399522960186, "metrics/particle/matched/delta_phi_abs_mean": 0.009019855409860611, "metrics/particle/matched/delta_r_mean": 0.013462131842970848, "metrics/particle/matched/energy_relative_abs_mean": 0.08553878217935562, "metrics/particle/matched/pid_accuracy": 0.9709617495536804, "metrics/particle/matched/pt_relative_abs_mean": 0.08032435923814774, "metrics/particle/matching/duplicate_fraction": 0.02799765020608902, "metrics/particle/matching/efficiency": 0.8423933386802673, "metrics/particle/matching/f1": 0.8451566696166992, "metrics/particle/matching/purity": 0.8479381799697876, "metrics/particle/matching/target_coverage_dr0p05": 0.8296392560005188, "metrics/particle/matching/target_coverage_dr0p10": 0.8749502301216125, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7438892126083374, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8750428557395935, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015209103003144264, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.082551009953022, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07883690297603607, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014997140504419804, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13091999292373657, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005114785395562649, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3503783047199249, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02108563855290413, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4828757047653198, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8407.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22789740562438965, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8426602482795715, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8045316934585571, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7012392282485962, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10777044296264648, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.823904037475586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013001523911952972, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6695058941841125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.039846036583185196, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.600898027420044, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9845.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8293380737304688, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.17701898515224457, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.8367156982421875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.543583631515503, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.02006347104907036, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.019640646874904633, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.07374906539917, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.20739223062992096, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.36933422088623, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02377074770629406, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09383657574653625, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09263550490140915, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01743995025753975, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14518749713897705, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023894863203167915, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9535314440727234, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12685441970825195, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0952813625335693, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1674.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7352407574653625, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.512614369392395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17415.455078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17415.302734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24696198105812073, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 879487.1875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07565660774707794, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6965818405151367, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0401490144431591, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.938746452331543, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2551.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6089487075805664, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.2184792757034302, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.917698860168457, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8019235134124756, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10188577324151993, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04718046635389328, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.7980570793151855, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.8697527647018433, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.050052642822266, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011779576539993286, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08725984394550323, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08104810863733292, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013384879566729069, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1346324235200882, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00039938095142133534, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1046638935804367, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005940939299762249, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4748431146144867, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20031.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13069243729114532, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.577725350856781, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4400930404663086, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.33503010869026184, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08518122881650925, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.773435592651367, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002270905766636133, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22003740072250366, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005648764315992594, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0611274242401123, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22458.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8761317133903503, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.18372011184692383, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.660829544067383, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.842679977416992, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.11646948009729385, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.09289608150720596, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.4678122997283936, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.31939810514450073, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 3.938173770904541, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013403688557446003, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08631079643964767, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08107493072748184, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014060438610613346, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13421939313411713, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030220509506762028, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22045578062534332, "metrics/particle/momentum/all/accepted/pz_bias": 0.002786782803013921, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.138651728630066, "metrics/particle/momentum/all/accepted/samples": 30112.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20239683985710144, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7209854125976562, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1275.166015625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1275.0574951171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1034027636051178, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 237935.046875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010673093609511852, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5282569527626038, "metrics/particle/momentum/all/assigned/pz_bias": 0.017833370715379715, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6924099922180176, "metrics/particle/momentum/all/assigned/samples": 34854.0, "metrics/particle/momentum/all/efficiency": 0.8423172831535339, "metrics/particle/momentum/all/event/energy_bias": -1.2251803874969482, "metrics/particle/momentum/all/event/p3_error_mean": 12.176609992980957, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.422834873199463, "metrics/particle/momentum/all/event/px_bias": 0.03464718163013458, "metrics/particle/momentum/all/event/py_bias": 0.12043590843677521, "metrics/particle/momentum/all/event/pz_abs_mean": 8.854839324951172, "metrics/particle/momentum/all/event/pz_bias": 1.396543025970459, "metrics/particle/momentum/all/event/pz_residual_rms": 16.463871002197266, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007580970413982868, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04329017549753189, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.035529185086488724, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004886720795184374, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08133086562156677, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015389350010082126, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13197720050811768, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0009278915822505951, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8736537098884583, "metrics/particle/momentum/class_1/accepted/samples": 16245.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06538263708353043, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18698856234550476, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22881560027599335, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1801813542842865, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.036989498883485794, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.166764259338379, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003792298724874854, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2412889152765274, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0030341886449605227, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6510478258132935, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9476171135902405, "metrics/particle/momentum/class_1/event/energy_bias": 1.3686490058898926, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.274856567382812, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.739943981170654, "metrics/particle/momentum/class_1/event/px_bias": 0.22483153641223907, "metrics/particle/momentum/class_1/event/py_bias": 0.25699520111083984, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.9555158615112305, "metrics/particle/momentum/class_1/event/pz_bias": 0.14322462677955627, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.032261848449707, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0325944721698761, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.190670445561409, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1918025016784668, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04363909736275673, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23186703026294708, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04117647185921669, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7130939960479736, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.051383763551712036, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.981822967529297, "metrics/particle/momentum/class_2/accepted/samples": 1360.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8026880025863647, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6831036806106567, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17125.267578125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17124.818359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35158056020736694, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 872167.1875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07594449073076248, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.878737211227417, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0960693284869194, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.688851833343506, "metrics/particle/momentum/class_2/assigned/samples": 2594.0, "metrics/particle/momentum/class_2/efficiency": 0.47058823704719543, "metrics/particle/momentum/class_2/event/energy_bias": -6.584874153137207, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.899658203125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.4648756980896, "metrics/particle/momentum/class_2/event/px_bias": 0.13392320275306702, "metrics/particle/momentum/class_2/event/py_bias": -0.07074147462844849, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.51118278503418, "metrics/particle/momentum/class_2/event/pz_bias": -0.28085410594940186, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.189071655273438, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01924978196620941, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13195064663887024, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13023057579994202, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03196973353624344, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17138925194740295, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007466401439160109, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1688038557767868, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.011690798215568066, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6194450259208679, "metrics/particle/momentum/class_3/accepted/samples": 12054.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2568860650062561, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2831521034240723, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1647089719772339, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0450758934020996, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1369149535894394, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.401859283447266, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007058178540319204, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.44570648670196533, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.017907826229929924, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.433819055557251, "metrics/particle/momentum/class_3/assigned/samples": 14593.0, "metrics/particle/momentum/class_3/efficiency": 0.7936006188392639, "metrics/particle/momentum/class_3/event/energy_bias": 5.058804035186768, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.720010757446289, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.773177623748779, "metrics/particle/momentum/class_3/event/px_bias": -0.34768933057785034, "metrics/particle/momentum/class_3/event/py_bias": -0.25959718227386475, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.198641777038574, "metrics/particle/momentum/class_3/event/pz_bias": 1.0755701065063477, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.20997142791748, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01127435639500618, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1484045833349228, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10557902604341507, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05788712203502655, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15981540083885193, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003508772002533078, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.38253507018089294, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08232251554727554, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2104849815368652, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28662729263305664, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.013924598693848, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1636111736297607, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.101454496383667, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15819208323955536, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.124642372131348, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01977401040494442, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7730575203895569, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.13493606448173523, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.0715131759643555, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.805084764957428, "metrics/particle/momentum/class_4/event/energy_bias": -1.14560067653656, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8402400016784668, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3800952434539795, "metrics/particle/momentum/class_4/event/px_bias": -0.05502140894532204, "metrics/particle/momentum/class_4/event/py_bias": 0.19509653747081757, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9836063981056213, "metrics/particle/momentum/class_4/event/pz_bias": 0.3348689675331116, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.1819539070129395, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00524072116240859, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021437738090753555, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020333047956228256, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008877123706042767, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05635945126414299, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1238253265619278, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.02671615220606327, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4638673663139343, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007538474164903164, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03642859309911728, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03407825529575348, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01027719583362341, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1034277081489563, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.17845478653907776, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06507626175880432, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.7517237663269043, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.07784102857112885, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7873109579086304, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6616543531417847, "metrics/particle/momentum/class_5/event/px_bias": 0.07860320806503296, "metrics/particle/momentum/class_5/event/py_bias": -0.001317223533987999, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3208114504814148, "metrics/particle/momentum/class_5/event/pz_bias": 0.12373354285955429, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.437928318977356, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5826053619384766, "step_valid_time": 17828.109285116196, "step_total_time": 17829.691890716553, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017968951563172, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06761742414707128, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2943, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9658680669510994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9690484030293053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9674556213017751, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.030951596970694766, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9654264076391176, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9638395792241946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9658680669510994, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001607579414686, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07454473948238671, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578577699736611, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9582784365393061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9580680570801318, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041721563460693896, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552238805970149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9556433904259991, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554335894621295, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578577699736611, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0068240596213682, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07052982956215059, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1244, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9438543247344461, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9402872260015117, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9420673987126088, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05971277399848828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9395313681027967, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9413101098068913, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9438543247344461} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_41000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..2b549219032f8e1db317bf15a1d49635196f3440 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9420234937476317, "train": {"Classification_binary": 2.468421697616577, "Classification": 0.0206815954297781, "Regression_pt": 0.13451556861400604, "Regression_eta": 0.010196697898209095, "Regression_sin_phi": 0.006388483569025993, "Regression_cos_phi": 0.005644918419420719, "Regression_energy": 0.13648340106010437, "Total": 7.838838577270508, "Total_unweighted": 2.7823323626071215}, "train_group": {}, "valid": {"Classification_binary": 2.351430575052897, "Classification": 0.017057381570339203, "Regression_pt": 0.12129586935043335, "Regression_eta": 0.010891047616799673, "Regression_sin_phi": 0.006358016282320023, "Regression_cos_phi": 0.006717930858333905, "Regression_energy": 0.12589352329572043, "Total": 7.386887232462565, "Total_unweighted": 2.6396443440268436, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016788117587566376, "diagnostic/loss/cld_pf/Classification_binary": 2.3535540103912354, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006606012117117643, "diagnostic/loss/cld_pf/Regression_energy": 0.12479609251022339, "diagnostic/loss/cld_pf/Regression_eta": 0.010140422731637955, "diagnostic/loss/cld_pf/Regression_pt": 0.1209266185760498, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006001732312142849, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08829095959663391, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18326693773269653, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06117168068885803, "diagnostic/regression/cld_pf/energy_residual_rms": 0.41003137826919556, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03820326551795006, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17903141677379608, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06537040323019028, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4027009904384613, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.948100745677948, "metrics/particle/class_1/pid_accuracy": 0.9875075817108154, "metrics/particle/class_2/efficiency": 0.46963655948638916, "metrics/particle/class_2/pid_accuracy": 0.7918485999107361, "metrics/particle/class_3/efficiency": 0.792545735836029, "metrics/particle/class_3/pid_accuracy": 0.9866071343421936, "metrics/particle/class_4/efficiency": 0.8153380155563354, "metrics/particle/class_4/pid_accuracy": 0.43316832184791565, "metrics/particle/class_5/efficiency": 0.9805825352668762, "metrics/particle/class_5/pid_accuracy": 0.6752475500106812, "metrics/particle/count/bias_mean": -0.41830065846443176, "metrics/particle/count/mae": 3.1215686798095703, "metrics/particle/count/prediction_mean": 70.14705657958984, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04723794385790825, "metrics/particle/event/energy_response_mean": 0.9967265725135803, "metrics/particle/event/met_abs_error": 4.642927169799805, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03865571692585945, "metrics/particle/event/scalar_pt_response_mean": 1.0049867630004883, "metrics/particle/event/vector_pt_closure": 0.041235025972127914, "metrics/particle/matched/delta_eta_abs_mean": 0.007704523392021656, "metrics/particle/matched/delta_phi_abs_mean": 0.008375155739486217, "metrics/particle/matched/delta_r_mean": 0.012805367819964886, "metrics/particle/matched/energy_relative_abs_mean": 0.08535044640302658, "metrics/particle/matched/pid_accuracy": 0.9716277122497559, "metrics/particle/matched/pt_relative_abs_mean": 0.08107954263687134, "metrics/particle/matching/duplicate_fraction": 0.028464943170547485, "metrics/particle/matching/efficiency": 0.8425785899162292, "metrics/particle/matching/f1": 0.8450833559036255, "metrics/particle/matching/purity": 0.8476030826568604, "metrics/particle/matching/target_coverage_dr0p05": 0.8317510485649109, "metrics/particle/matching/target_coverage_dr0p10": 0.8752188086509705, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.74174964427948, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8752836585044861, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014449039474129677, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08166582882404327, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0791136771440506, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019157975912094116, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13130125403404236, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004776689689606428, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3472628593444824, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.009674547240138054, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4757391214370728, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8374.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23251742124557495, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9278674125671387, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8897200226783752, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7892419695854187, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11158623546361923, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.438263893127441, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013337405398488045, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.666788637638092, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.006949049420654774, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4774374961853027, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9822.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8260826468467712, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.2845616042613983, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.051997184753418, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7099246978759766, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.05203366279602051, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.01948380470275879, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.184556007385254, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3948475420475006, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.648706436157227, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022924527525901794, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08893553167581558, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0894177109003067, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015336710959672928, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14063289761543274, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02360774762928486, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9412428140640259, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18943347036838531, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0577621459960938, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1652.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7015302181243896, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.230393409729004, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9135.6044921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9135.4580078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24351924657821655, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 460841.90625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0710919052362442, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5522994995117188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06063907966017723, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.193803310394287, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2546.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.600945770740509, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.661252498626709, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.107874870300293, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8474447727203369, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09928584098815918, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.052663277834653854, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.987497329711914, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5454371571540833, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.48223876953125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01106338668614626, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0877075344324112, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08250277489423752, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015779592096805573, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13687197864055634, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003980891779065132, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1037527471780777, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010676744394004345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.46312668919563293, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20096.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12785494327545166, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6360794305801392, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4818858206272125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3774750530719757, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08337777853012085, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.839745998382568, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002266666619107127, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22728298604488373, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03439166024327278, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.579166293144226, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22500.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8789747357368469, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.28509601950645447, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.810471534729004, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.878897666931152, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.06165587157011032, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.29740598797798157, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.6809535026550293, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7540625333786011, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.891058444976807, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012655115686357021, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08609527349472046, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08193983882665634, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016694504767656326, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1355576068162918, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028882543556392193, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21738022565841675, "metrics/particle/momentum/all/accepted/pz_bias": -0.0005766408867202699, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1230825185775757, "metrics/particle/momentum/all/accepted/samples": 30122.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19922618567943573, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7616692185401917, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 667.6273803710938, "metrics/particle/momentum/all/assigned/pt_relative_bias": 667.5209350585938, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10301709175109863, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 124528.1875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010410691611468792, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5208563208580017, "metrics/particle/momentum/all/assigned/pz_bias": 0.0197223462164402, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.667750120162964, "metrics/particle/momentum/all/assigned/samples": 34868.0, "metrics/particle/momentum/all/efficiency": 0.8425970077514648, "metrics/particle/momentum/all/event/energy_bias": -1.0915948152542114, "metrics/particle/momentum/all/event/p3_error_mean": 12.466906547546387, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.569421291351318, "metrics/particle/momentum/all/event/px_bias": 0.014403678476810455, "metrics/particle/momentum/all/event/py_bias": 0.3695530295372009, "metrics/particle/momentum/all/event/pz_abs_mean": 9.092437744140625, "metrics/particle/momentum/all/event/pz_bias": 1.6943471431732178, "metrics/particle/momentum/all/event/pz_residual_rms": 16.7731990814209, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007661742158234119, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04295341670513153, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03613690286874771, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007222191896289587, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08279971778392792, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011698787566274405, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1285775750875473, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.002504491014406085, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8214865326881409, "metrics/particle/momentum/class_1/accepted/samples": 16241.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06807395070791245, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20798279345035553, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22549428045749664, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17856942117214203, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.037162359803915024, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.027897596359253, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003850417211651802, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.25006991624832153, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0038795522414147854, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6798158884048462, "metrics/particle/momentum/class_1/assigned/samples": 17141.0, "metrics/particle/momentum/class_1/efficiency": 0.9473837614059448, "metrics/particle/momentum/class_1/event/energy_bias": 1.3751968145370483, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.391839981079102, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.866167068481445, "metrics/particle/momentum/class_1/event/px_bias": 0.3410056531429291, "metrics/particle/momentum/class_1/event/py_bias": 0.17819133400917053, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.031438827514648, "metrics/particle/momentum/class_1/event/pz_bias": 0.32049983739852905, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.139967918395996, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032324016094207764, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18931816518306732, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19104816019535065, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03833674639463425, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23199895024299622, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04444444552063942, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7532989978790283, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17210187017917633, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.113592624664307, "metrics/particle/momentum/class_2/accepted/samples": 1350.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7966914772987366, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6924285292625427, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 8968.6767578125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 8968.236328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3494022488594055, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 456646.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07288854569196701, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.814365863800049, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.09046869724988937, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.5510358810424805, "metrics/particle/momentum/class_2/assigned/samples": 2593.0, "metrics/particle/momentum/class_2/efficiency": 0.46712803840637207, "metrics/particle/momentum/class_2/event/energy_bias": -6.046360969543457, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.59299087524414, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.425521373748779, "metrics/particle/momentum/class_2/event/px_bias": -0.08588752895593643, "metrics/particle/momentum/class_2/event/py_bias": 0.32957780361175537, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.210147857666016, "metrics/particle/momentum/class_2/event/pz_bias": -0.2820040285587311, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.888452529907227, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017307566478848457, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13189931213855743, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.131403386592865, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03464609384536743, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17283415794372559, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005795181496068835, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16126561164855957, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.011521345004439354, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5537475347518921, "metrics/particle/momentum/class_3/accepted/samples": 12079.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24739079177379608, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3474862575531006, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.229694128036499, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1103453636169434, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13602136075496674, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.538894653320312, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0071193864569067955, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4333900809288025, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.023649703711271286, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4426000118255615, "metrics/particle/momentum/class_3/assigned/samples": 14608.0, "metrics/particle/momentum/class_3/efficiency": 0.7952465415000916, "metrics/particle/momentum/class_3/event/energy_bias": 4.579433441162109, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.434676170349121, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.78054666519165, "metrics/particle/momentum/class_3/event/px_bias": -0.2494077980518341, "metrics/particle/momentum/class_3/event/py_bias": -0.28317728638648987, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.881053924560547, "metrics/particle/momentum/class_3/event/pz_bias": 1.2297474145889282, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.056943893432617, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01114656776189804, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15233951807022095, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11498073488473892, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06246793642640114, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16946592926979065, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0035211266949772835, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4287761151790619, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07964907586574554, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2501107454299927, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27201446890830994, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.267127990722656, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.4550368785858154, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.3981144428253174, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16430595517158508, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 36.150962829589844, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01133144460618496, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6206955313682556, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08672700077295303, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9176461696624756, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -1.1091234683990479, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8915810585021973, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4093339443206787, "metrics/particle/momentum/class_4/event/px_bias": -0.07482565194368362, "metrics/particle/momentum/class_4/event/py_bias": 0.13965044915676117, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0266547203063965, "metrics/particle/momentum/class_4/event/pz_bias": 0.2971256375312805, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.241432189941406, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005368303507566452, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02202119491994381, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02084207907319069, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00654620723798871, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05644780024886131, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1371772438287735, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.07443961501121521, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4959558844566345, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02332983911037445, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04048982635140419, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03672913834452629, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01580338180065155, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11614777147769928, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.15638233721256256, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.060719769448041916, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.5292404890060425, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.10925932228565216, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.852233350276947, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6974174380302429, "metrics/particle/momentum/class_5/event/px_bias": 0.08351898193359375, "metrics/particle/momentum/class_5/event/py_bias": 0.005310775712132454, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3760736286640167, "metrics/particle/momentum/class_5/event/pz_bias": 0.12897837162017822, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.650497555732727, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.7232890129089355, "step_valid_time": 18275.98410511017, "step_total_time": 18277.707394361496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026597258776953, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06541924656271181, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9655398752871677, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9671268902038133, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.966332731154541, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03287310979618672, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9619297669839186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9635108481262328, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9627196584003942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9655398752871677, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003481638221407, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07490651874577092, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2184, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9587357330992098, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9591567852437418, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9589462129527991, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04084321475625823, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547848990342406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9552042160737813, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9549945115257958, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9587357330992098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067995308772906, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07603761597528602, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1324, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1240, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9388217522658611, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9409538228614686, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06117824773413897, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9408194233687405, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9365558912386707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9386828160484482, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9430955993930197} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_42000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..a127a788bc6015e3aef639875942004150ddd778 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9893899204244032, "train": {"Classification_binary": 2.345905065536499, "Classification": 0.01699052006006241, "Regression_pt": 0.11946730315685272, "Regression_eta": 0.011182751506567001, "Regression_sin_phi": 0.006675136741250753, "Regression_cos_phi": 0.00616078358143568, "Regression_energy": 0.1244654655456543, "Total": 7.335835933685303, "Total_unweighted": 2.630847026128322}, "train_group": {}, "valid": {"Classification_binary": 2.347668488820394, "Classification": 0.01698342834909757, "Regression_pt": 0.12042904893557231, "Regression_eta": 0.010983226199944815, "Regression_sin_phi": 0.006366248552997907, "Regression_cos_phi": 0.006721671670675278, "Regression_energy": 0.12506951888402304, "Total": 7.3655853271484375, "Total_unweighted": 2.634221631412705, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016731029376387596, "diagnostic/loss/cld_pf/Classification_binary": 2.3536744117736816, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0065664867870509624, "diagnostic/loss/cld_pf/Regression_energy": 0.12432718276977539, "diagnostic/loss/cld_pf/Regression_eta": 0.010156038217246532, "diagnostic/loss/cld_pf/Regression_pt": 0.12041962891817093, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.0060207596980035305, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08671660721302032, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18289673328399658, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05959732085466385, "diagnostic/regression/cld_pf/energy_residual_rms": 0.40759724378585815, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03468857705593109, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17847563326358795, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06185571104288101, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40068450570106506, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9487332105636597, "metrics/particle/class_1/pid_accuracy": 0.9878188967704773, "metrics/particle/class_2/efficiency": 0.4722570478916168, "metrics/particle/class_2/pid_accuracy": 0.7908323407173157, "metrics/particle/class_3/efficiency": 0.7917326092720032, "metrics/particle/class_3/pid_accuracy": 0.9863435626029968, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.41198045015335083, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6858847141265869, "metrics/particle/count/bias_mean": -0.42091503739356995, "metrics/particle/count/mae": 2.9712417125701904, "metrics/particle/count/prediction_mean": 70.14444732666016, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04511666297912598, "metrics/particle/event/energy_response_mean": 0.9938433766365051, "metrics/particle/event/met_abs_error": 4.317571640014648, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0357794351875782, "metrics/particle/event/scalar_pt_response_mean": 0.9987861514091492, "metrics/particle/event/vector_pt_closure": 0.04015328735113144, "metrics/particle/matched/delta_eta_abs_mean": 0.007420788519084454, "metrics/particle/matched/delta_phi_abs_mean": 0.008467722684144974, "metrics/particle/matched/delta_r_mean": 0.012644603848457336, "metrics/particle/matched/energy_relative_abs_mean": 0.08573487401008606, "metrics/particle/matched/pid_accuracy": 0.9714163541793823, "metrics/particle/matched/pt_relative_abs_mean": 0.08098634332418442, "metrics/particle/matching/duplicate_fraction": 0.02731059119105339, "metrics/particle/matching/efficiency": 0.8428286910057068, "metrics/particle/matching/f1": 0.845349907875061, "metrics/particle/matching/purity": 0.8478862643241882, "metrics/particle/matching/target_coverage_dr0p05": 0.8306303024291992, "metrics/particle/matching/target_coverage_dr0p10": 0.8744222521781921, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7435465455055237, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8771824240684509, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01456562802195549, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08163495361804962, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07828017324209213, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017745889723300934, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12959393858909607, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004901374690234661, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3440227806568146, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.002510111778974533, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4602789878845215, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8365.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23259207606315613, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8036609888076782, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7487486004829407, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6479865908622742, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10896719992160797, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.1303839683532715, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012389560230076313, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6376654505729675, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.015501157380640507, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.357708692550659, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9847.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8251948356628418, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5324217677116394, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.252209663391113, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.75508451461792, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.01760340854525566, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.05989154428243637, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.410002708435059, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5087937712669373, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.930665016174316, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024118147790431976, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0929761677980423, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09106194227933884, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018691541627049446, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14209894835948944, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023650696501135826, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9473667740821838, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18456031382083893, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0079734325408936, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1649.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7025801539421082, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6655611991882324, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6034.72021484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6034.56640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25058457255363464, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 305548.625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07599376142024994, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.600140333175659, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09398204833269119, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.336536884307861, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2566.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5998544692993164, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4028579890727997, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.090548515319824, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8628888726234436, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03723859786987305, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07157254219055176, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.955563545227051, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.1846899390220642, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.208402633666992, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010800238698720932, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08821950107812881, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08256921917200089, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013425500132143497, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13722944259643555, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003979307657573372, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10552868247032166, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.006329254247248173, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.47276368737220764, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20104.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13023120164871216, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5882367491722107, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4462023973464966, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.33958369493484497, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08072267472743988, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.9929728507995605, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002358490601181984, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21907857060432434, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01339549757540226, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0870462656021118, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22472.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8793246746063232, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.874210774898529, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.879182815551758, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.9843831062316895, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4601227045059204, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.08453820645809174, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.608335256576538, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.061623506247997284, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.852077960968018, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012575213797390461, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08665113151073456, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08184295892715454, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01491377130150795, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13542725145816803, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002921840874478221, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21785996854305267, "metrics/particle/momentum/all/accepted/pz_bias": -0.005182931665331125, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.112130880355835, "metrics/particle/momentum/all/accepted/samples": 30118.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20122432708740234, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7282882332801819, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 444.3885192871094, "metrics/particle/momentum/all/assigned/pt_relative_bias": 444.2799987792969, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10118962079286575, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 82868.46875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010606277734041214, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5123744606971741, "metrics/particle/momentum/all/assigned/pz_bias": -0.00265942863188684, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.50787091255188, "metrics/particle/momentum/all/assigned/samples": 34885.0, "metrics/particle/momentum/all/efficiency": 0.8424851298332214, "metrics/particle/momentum/all/event/energy_bias": -1.809490442276001, "metrics/particle/momentum/all/event/p3_error_mean": 12.342782020568848, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.412956237792969, "metrics/particle/momentum/all/event/px_bias": -0.44048750400543213, "metrics/particle/momentum/all/event/py_bias": -0.07285722345113754, "metrics/particle/momentum/all/event/pz_abs_mean": 8.995723724365234, "metrics/particle/momentum/all/event/pz_bias": -0.26248031854629517, "metrics/particle/momentum/all/event/pz_residual_rms": 16.02479362487793, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007463323418051004, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04342225566506386, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.036196157336235046, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005376179702579975, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0821932777762413, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012932626996189356, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13022850453853607, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007500080391764641, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8398342132568359, "metrics/particle/momentum/class_1/accepted/samples": 16238.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0672755017876625, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19196738302707672, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.23181557655334473, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18266405165195465, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03734173625707626, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.4808430671691895, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00373417348600924, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2374771684408188, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.016544565558433533, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.418389916419983, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9472087621688843, "metrics/particle/momentum/class_1/event/energy_bias": 1.445022702217102, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.854907035827637, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.591554164886475, "metrics/particle/momentum/class_1/event/px_bias": 0.3057117462158203, "metrics/particle/momentum/class_1/event/py_bias": 0.08784845471382141, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.55388069152832, "metrics/particle/momentum/class_1/event/pz_bias": -0.660877525806427, "metrics/particle/momentum/class_1/event/pz_residual_rms": 9.748418807983398, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03203785419464111, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19012261927127838, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19172441959381104, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04897356405854225, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23214657604694366, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04219096899032593, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7105811834335327, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.18016046285629272, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9369924068450928, "metrics/particle/momentum/class_2/accepted/samples": 1351.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7941157817840576, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6758427619934082, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5944.78662109375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5944.33544921875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3410138189792633, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 303311.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.074884794652462, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7673587799072266, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13215996325016022, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.933187961578369, "metrics/particle/momentum/class_2/assigned/samples": 2604.0, "metrics/particle/momentum/class_2/efficiency": 0.4674740433692932, "metrics/particle/momentum/class_2/event/energy_bias": -6.92603874206543, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.983908653259277, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.500788688659668, "metrics/particle/momentum/class_2/event/px_bias": -0.03124411404132843, "metrics/particle/momentum/class_2/event/py_bias": -0.0798531174659729, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.552596092224121, "metrics/particle/momentum/class_2/event/pz_bias": -1.6638813018798828, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.281570434570312, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017387907952070236, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13242487609386444, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13113579154014587, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03396892175078392, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1730382740497589, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006626905524171889, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16546110808849335, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.013961479999125004, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6014271378517151, "metrics/particle/momentum/class_3/accepted/samples": 12072.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2534617483615875, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2918236255645752, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.177846074104309, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.058690071105957, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13286809623241425, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.385762214660645, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007252326235175133, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4359579384326935, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.031270481646060944, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5097644329071045, "metrics/particle/momentum/class_3/assigned/samples": 14616.0, "metrics/particle/momentum/class_3/efficiency": 0.7947856783866882, "metrics/particle/momentum/class_3/event/energy_bias": 5.049723148345947, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.490105628967285, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.71064567565918, "metrics/particle/momentum/class_3/event/px_bias": -0.5846990346908569, "metrics/particle/momentum/class_3/event/py_bias": -0.34823548793792725, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.025372505187988, "metrics/particle/momentum/class_3/event/pz_bias": 1.4791631698608398, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.238670349121094, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011727435514330864, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15590058267116547, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10768719017505646, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05979490280151367, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1620776206254959, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0069204154424369335, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3841722011566162, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10487889498472214, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1758917570114136, "metrics/particle/momentum/class_4/accepted/samples": 289.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.255695104598999, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.158941268920898, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.041717529296875, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.987863063812256, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1614730805158615, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.60282897949219, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014164306223392487, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5496391654014587, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.19911928474903107, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7597944736480713, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8163841962814331, "metrics/particle/momentum/class_4/event/energy_bias": -1.428663730621338, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8047199249267578, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.417264461517334, "metrics/particle/momentum/class_4/event/px_bias": -0.17459365725517273, "metrics/particle/momentum/class_4/event/py_bias": 0.25490301847457886, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8956745862960815, "metrics/particle/momentum/class_4/event/pz_bias": 0.47032877802848816, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.3715531826019287, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005783609114587307, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.024548832327127457, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023694179952144623, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004073052201420069, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06978344917297363, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1630304902791977, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.060896407812833786, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7375373840332031, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.022780682891607285, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03994735702872276, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.037141118198633194, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.011645529419183731, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11347031593322754, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.18426702916622162, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.04386797547340393, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.758160412311554, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.05046641081571579, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7303234934806824, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6065971255302429, "metrics/particle/momentum/class_5/event/px_bias": 0.044337525963783264, "metrics/particle/momentum/class_5/event/py_bias": 0.012479906901717186, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3109942078590393, "metrics/particle/momentum/class_5/event/pz_bias": 0.11278644949197769, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4398257732391357, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6289842128753662, "step_valid_time": 18723.989535093307, "step_total_time": 18725.6185195446, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990202617579453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06684548104434163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2936, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9635707253035772, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9686572088419664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9661072721289898, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03134279115803365, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596324253363965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9646981194325305, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9621586048042119, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9635707253035772, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991308635427771, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07380680736712653, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2178, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9591747146619842, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958753839403247, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9589642308536318, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041246160596752963, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561018437225637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9556823168056165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.955892034233048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9591747146619842, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032384188765398, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07191201798359181, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9385432473444613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061456752655538696, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9370257966616085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385432473444613} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_43000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..891439df9fb4b5f2d11593b665fdb6292cb74c5b --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0367563471011745, "train": {"Classification_binary": 2.391493082046509, "Classification": 0.01545021403580904, "Regression_pt": 0.1192067563533783, "Regression_eta": 0.010026762261986732, "Regression_sin_phi": 0.007701055146753788, "Regression_cos_phi": 0.006945435423403978, "Regression_energy": 0.12275025248527527, "Total": 7.351268768310547, "Total_unweighted": 2.673573557753116}, "train_group": {}, "valid": {"Classification_binary": 2.3347670237223306, "Classification": 0.01691219707330068, "Regression_pt": 0.12055838108062744, "Regression_eta": 0.011106240252653757, "Regression_sin_phi": 0.0063619545350472135, "Regression_cos_phi": 0.006716296697656314, "Regression_energy": 0.1253070334593455, "Total": 7.358930587768555, "Total_unweighted": 2.6217291268209615, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016759220510721207, "diagnostic/loss/cld_pf/Classification_binary": 2.338285446166992, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006569796707481146, "diagnostic/loss/cld_pf/Regression_energy": 0.1244521290063858, "diagnostic/loss/cld_pf/Regression_eta": 0.010189269669353962, "diagnostic/loss/cld_pf/Regression_pt": 0.12048669904470444, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005995988380163908, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08960020542144775, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18244892358779907, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06248091906309128, "diagnostic/regression/cld_pf/energy_residual_rms": 0.40976497530937195, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03830553963780403, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17789147794246674, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06547267735004425, "diagnostic/regression/cld_pf/pt_residual_rms": 0.40186607837677, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9490973353385925, "metrics/particle/class_1/pid_accuracy": 0.9871976375579834, "metrics/particle/class_2/efficiency": 0.4720291793346405, "metrics/particle/class_2/pid_accuracy": 0.7914554476737976, "metrics/particle/class_3/efficiency": 0.7926116585731506, "metrics/particle/class_3/pid_accuracy": 0.9868855476379395, "metrics/particle/class_4/efficiency": 0.82542884349823, "metrics/particle/class_4/pid_accuracy": 0.4278728663921356, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.7145708799362183, "metrics/particle/count/bias_mean": -0.27450981736183167, "metrics/particle/count/mae": 2.983006477355957, "metrics/particle/count/prediction_mean": 70.29084777832031, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04635554179549217, "metrics/particle/event/energy_response_mean": 0.9978705644607544, "metrics/particle/event/met_abs_error": 4.705942630767822, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03730856254696846, "metrics/particle/event/scalar_pt_response_mean": 1.0059441328048706, "metrics/particle/event/vector_pt_closure": 0.0415385439991951, "metrics/particle/matched/delta_eta_abs_mean": 0.007535780780017376, "metrics/particle/matched/delta_phi_abs_mean": 0.008219609968364239, "metrics/particle/matched/delta_r_mean": 0.01257333904504776, "metrics/particle/matched/energy_relative_abs_mean": 0.08504939824342728, "metrics/particle/matched/pid_accuracy": 0.971642255783081, "metrics/particle/matched/pt_relative_abs_mean": 0.0803043320775032, "metrics/particle/matching/duplicate_fraction": 0.028490399941802025, "metrics/particle/matching/efficiency": 0.843338131904602, "metrics/particle/matching/f1": 0.8449816703796387, "metrics/particle/matching/purity": 0.846631646156311, "metrics/particle/matching/target_coverage_dr0p05": 0.8325476050376892, "metrics/particle/matching/target_coverage_dr0p10": 0.8760431408882141, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7441763281822205, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8764044046401978, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014055310748517513, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08130715042352676, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07833462208509445, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01709035411477089, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13019819557666779, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004657828714698553, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3507169783115387, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00010038034088211134, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5083876848220825, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8373.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22973689436912537, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8708975315093994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8059476017951965, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7061545252799988, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11043380945920944, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.010035991668701, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01310576032847166, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6882522106170654, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02805192396044731, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7236318588256836, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9843.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.825984001159668, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.037984222173690796, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.988528251647949, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6187286376953125, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.0939566120505333, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.053091417998075485, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.194466590881348, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.11305296421051025, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.621574401855469, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02285034954547882, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0893583744764328, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08878747373819351, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01843435876071453, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13940131664276123, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022519780322909355, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9516855478286743, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.14369863271713257, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.143524646759033, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1643.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7348142266273499, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2130831480026245, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17565.80078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17565.638671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2499019205570221, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 886732.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07336210459470749, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.55456280708313, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.11374698579311371, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.985273838043213, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2549.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5976718664169312, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.7606428861618042, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.0755615234375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8433048725128174, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06052102893590927, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0928267315030098, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.95186185836792, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.31474238634109497, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.980976104736328, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010863171890377998, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08741758018732071, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08161410689353943, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016338437795639038, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13581116497516632, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00029829968116246164, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1038745567202568, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0044809565879404545, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4656265676021576, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20114.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12963372468948364, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6266021728515625, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4745388627052307, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3722808361053467, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08270042389631271, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.66102409362793, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0019986676052212715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2233544886112213, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013725061900913715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5033864974975586, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22515.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8797620534896851, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5417320728302002, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.480099678039551, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.64821720123291, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3145791292190552, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.11679808795452118, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.4162538051605225, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.12437642365694046, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.091647624969482, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012403919361531734, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08582533895969391, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0810939222574234, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016661683097481728, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13447536528110504, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002721539931371808, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21870246529579163, "metrics/particle/momentum/all/accepted/pz_bias": -0.004816674627363682, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1471132040023804, "metrics/particle/momentum/all/accepted/samples": 30130.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20205245912075043, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7383143901824951, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1283.23388671875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1283.1279296875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10273011028766632, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 239619.078125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010341765359044075, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5246762633323669, "metrics/particle/momentum/all/assigned/pz_bias": 0.00845657754689455, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6669528484344482, "metrics/particle/momentum/all/assigned/samples": 34907.0, "metrics/particle/momentum/all/efficiency": 0.8428207635879517, "metrics/particle/momentum/all/event/energy_bias": -1.2568951845169067, "metrics/particle/momentum/all/event/p3_error_mean": 12.223305702209473, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.2856669425964355, "metrics/particle/momentum/all/event/px_bias": 0.1601014882326126, "metrics/particle/momentum/all/event/py_bias": 0.02912006340920925, "metrics/particle/momentum/all/event/pz_abs_mean": 9.062113761901855, "metrics/particle/momentum/all/event/pz_bias": 0.5521717667579651, "metrics/particle/momentum/all/event/pz_residual_rms": 16.26885414123535, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0074152578599750996, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04240882396697998, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03531104326248169, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006874468177556992, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0811329111456871, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011687992373481393, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1292986124753952, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.001196567784063518, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8566220998764038, "metrics/particle/momentum/class_1/accepted/samples": 16256.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06488630920648575, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20799484848976135, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2247079312801361, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17978033423423767, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03658322989940643, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.890601873397827, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0040842522867023945, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24670399725437164, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008919168263673782, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6835830211639404, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9482587575912476, "metrics/particle/momentum/class_1/event/energy_bias": 1.5351040363311768, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.351780891418457, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.818352222442627, "metrics/particle/momentum/class_1/event/px_bias": 0.1907048225402832, "metrics/particle/momentum/class_1/event/py_bias": -0.027752012014389038, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.018733978271484, "metrics/particle/momentum/class_1/event/pz_bias": 0.1970914602279663, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.033989906311035, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0318385474383831, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19164852797985077, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1934925615787506, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03825473040342331, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23344282805919647, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03979366272687912, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7419229745864868, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17973007261753082, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.119442939758301, "metrics/particle/momentum/class_2/accepted/samples": 1357.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8006775975227356, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6260805130004883, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17206.4375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17205.98828125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3416602611541748, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 877655.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07071483135223389, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.775432586669922, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.08382359892129898, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.311161041259766, "metrics/particle/momentum/class_2/assigned/samples": 2602.0, "metrics/particle/momentum/class_2/efficiency": 0.46955016255378723, "metrics/particle/momentum/class_2/event/energy_bias": -6.604297637939453, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.920524597167969, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.551970481872559, "metrics/particle/momentum/class_2/event/px_bias": 0.008969366550445557, "metrics/particle/momentum/class_2/event/py_bias": 0.0990772545337677, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.49467945098877, "metrics/particle/momentum/class_2/event/pz_bias": -1.2833067178726196, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.937760353088379, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01707232929766178, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1317165046930313, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13037735223770142, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03511616960167885, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17177382111549377, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006632399163208902, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16567128896713257, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00785186979919672, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5948681235313416, "metrics/particle/momentum/class_3/accepted/samples": 12062.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2568381428718567, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3021206855773926, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.204602599143982, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.086288571357727, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13744108378887177, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.396552085876465, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007035999558866024, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4517126977443695, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019765645265579224, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.521613121032715, "metrics/particle/momentum/class_3/assigned/samples": 14639.0, "metrics/particle/momentum/class_3/efficiency": 0.794127345085144, "metrics/particle/momentum/class_3/event/energy_bias": 4.82681941986084, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.631026268005371, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.842541694641113, "metrics/particle/momentum/class_3/event/px_bias": -0.18934029340744019, "metrics/particle/momentum/class_3/event/py_bias": -0.1679895967245102, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.038163185119629, "metrics/particle/momentum/class_3/event/pz_bias": 1.2353779077529907, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.1190767288208, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011397228576242924, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15452739596366882, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10778120905160904, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06001165881752968, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16220985352993011, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0034722222480922937, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3659104108810425, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0589991994202137, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.149905800819397, "metrics/particle/momentum/class_4/accepted/samples": 288.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27267012000083923, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.26457405090332, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.379796028137207, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.3240718841552734, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1497175097465515, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 37.74361038208008, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6135315299034119, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.22063495218753815, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.826159954071045, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8135592937469482, "metrics/particle/momentum/class_4/event/energy_bias": -1.2109956741333008, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9386279582977295, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4892864227294922, "metrics/particle/momentum/class_4/event/px_bias": -0.005526697263121605, "metrics/particle/momentum/class_4/event/py_bias": 0.1546589583158493, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9982792735099792, "metrics/particle/momentum/class_4/event/pz_bias": 0.27745679020881653, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.228496074676514, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004634188022464514, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019068026915192604, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.018691249191761017, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007917347364127636, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05020631477236748, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12052889168262482, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03902796283364296, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4969276189804077, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007034435402601957, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04075995832681656, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.038346439599990845, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.02090313658118248, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12159527838230133, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20304515957832336, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04056035727262497, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.935817301273346, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.19647493958473206, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.764455258846283, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6317291855812073, "metrics/particle/momentum/class_5/event/px_bias": 0.1552942395210266, "metrics/particle/momentum/class_5/event/py_bias": -0.028874551877379417, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.32974112033843994, "metrics/particle/momentum/class_5/event/pz_bias": 0.12555232644081116, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.506155252456665, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6790111064910889, "step_valid_time": 19171.324108362198, "step_total_time": 19173.003119945526, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032568509591226, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0682478327763052, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2945, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2936, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9665244502789629, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9674770039421814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9670004925299622, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03252299605781866, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9635707253035772, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9645203679369251, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.964045312756526, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9665244502789629, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033634694709102, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07389145369884864, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2187, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2178, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9600526777875329, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9604743083003953, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9602634467618002, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039525691699604744, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561018437225637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9565217391304348, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9563117453347969, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9600526777875329, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0049354421228718, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07654614338281374, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1239, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9415781487101669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9408642911296436, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9412210845657945, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05913570887035633, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9400606980273141, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9393479909021987, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9397042093287827, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9415781487101669} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_44000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..d7b74876a808fe42bf86041cd8f3c18a413efe17 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0841227737779464, "train": {"Classification_binary": 2.311857223510742, "Classification": 0.014947396703064442, "Regression_pt": 0.12930473685264587, "Regression_eta": 0.024001454934477806, "Regression_sin_phi": 0.0066110375337302685, "Regression_cos_phi": 0.005615150090306997, "Regression_energy": 0.13093166053295135, "Total": 7.983541488647461, "Total_unweighted": 2.623268660157919}, "train_group": {}, "valid": {"Classification_binary": 2.329636255900065, "Classification": 0.01692073916395505, "Regression_pt": 0.12064411242802937, "Regression_eta": 0.010951260725657145, "Regression_sin_phi": 0.006389676903684934, "Regression_cos_phi": 0.006742465620239575, "Regression_energy": 0.1254684031009674, "Total": 7.352966944376628, "Total_unweighted": 2.6167529138425984, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016796918585896492, "diagnostic/loss/cld_pf/Classification_binary": 2.3354177474975586, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006588890682905912, "diagnostic/loss/cld_pf/Regression_energy": 0.12422975897789001, "diagnostic/loss/cld_pf/Regression_eta": 0.010121642611920834, "diagnostic/loss/cld_pf/Regression_pt": 0.1201736330986023, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006002689711749554, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08674473315477371, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1818632036447525, "diagnostic/regression/cld_pf/energy_residual_mean": 0.059625450521707535, "diagnostic/regression/cld_pf/energy_residual_rms": 0.40566885471343994, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03308020904660225, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1765611320734024, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06024735048413277, "diagnostic/regression/cld_pf/pt_residual_rms": 0.39789891242980957, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9503622055053711, "metrics/particle/class_1/pid_accuracy": 0.9873961210250854, "metrics/particle/class_2/efficiency": 0.47749799489974976, "metrics/particle/class_2/pid_accuracy": 0.7926509380340576, "metrics/particle/class_3/efficiency": 0.798655092716217, "metrics/particle/class_3/pid_accuracy": 0.9863793849945068, "metrics/particle/class_4/efficiency": 0.8314833641052246, "metrics/particle/class_4/pid_accuracy": 0.4223301112651825, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.675298810005188, "metrics/particle/count/bias_mean": -0.012418300844728947, "metrics/particle/count/mae": 2.9901959896087646, "metrics/particle/count/prediction_mean": 70.55294036865234, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.0453735813498497, "metrics/particle/event/energy_response_mean": 0.9963959455490112, "metrics/particle/event/met_abs_error": 4.5488104820251465, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03582455590367317, "metrics/particle/event/scalar_pt_response_mean": 1.0014821290969849, "metrics/particle/event/vector_pt_closure": 0.03986094519495964, "metrics/particle/matched/delta_eta_abs_mean": 0.0074087572284042835, "metrics/particle/matched/delta_phi_abs_mean": 0.008518459275364876, "metrics/particle/matched/delta_r_mean": 0.012698705308139324, "metrics/particle/matched/energy_relative_abs_mean": 0.08618999272584915, "metrics/particle/matched/pid_accuracy": 0.9712620377540588, "metrics/particle/matched/pt_relative_abs_mean": 0.08050554990768433, "metrics/particle/matching/duplicate_fraction": 0.027560075744986534, "metrics/particle/matching/efficiency": 0.8470059633255005, "metrics/particle/matching/f1": 0.8470805287361145, "metrics/particle/matching/purity": 0.8471550345420837, "metrics/particle/matching/target_coverage_dr0p05": 0.834918737411499, "metrics/particle/matching/target_coverage_dr0p10": 0.8783494830131531, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7472236156463623, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.879673957824707, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01447984203696251, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08288528025150299, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07906129956245422, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016546085476875305, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13095781207084656, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00498575484380126, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3517993986606598, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01664326898753643, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4879848957061768, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8424.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23101580142974854, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7762424349784851, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7345227599143982, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6320222020149231, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10861650854349136, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.901340484619141, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012641585431993008, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6573153138160706, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.03572932258248329, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5570924282073975, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9888.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8310151100158691, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.34155184030532837, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.925717353820801, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5554380416870117, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1209292858839035, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.058335624635219574, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.15988826751709, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7989775538444519, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.366504669189453, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022909507155418396, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09389321506023407, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09163004159927368, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015777578577399254, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1432899385690689, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02139037474989891, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9518170952796936, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.213790163397789, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.161168336868286, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1683.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7180769443511963, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5576008558273315, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5878.7197265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5878.56103515625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24437548220157623, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 298357.28125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07525213062763214, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.685946464538574, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2660525441169739, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.974368095397949, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2578.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6122226119041443, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.27191960811615, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.87209415435791, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.7665570378303528, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.042518630623817444, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.028707757592201233, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.764771938323975, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7687639594078064, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.571533203125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011031643487513065, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08824189752340317, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08155277371406555, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013016819953918457, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1353946477174759, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003469812509138137, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10388047993183136, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.003198959631845355, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.47032615542411804, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20174.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12866878509521484, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5973361134529114, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.44279491901397705, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.336616575717926, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.081900954246521, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.993843078613281, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021319121588021517, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21603518724441528, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007371292915195227, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0931241512298584, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22515.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8823863863945007, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.4338824152946472, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.6068220138549805, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.777309894561768, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.31109434366226196, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.061362721025943756, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.462780475616455, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.03782859444618225, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.639133930206299, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012651077471673489, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08706581592559814, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08141974359750748, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014152081683278084, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.134630024433136, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002807040698826313, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21997791528701782, "metrics/particle/momentum/all/accepted/pz_bias": -0.01864362321794033, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1483585834503174, "metrics/particle/momentum/all/accepted/samples": 30281.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20103669166564941, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.718675971031189, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 433.737548828125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 433.62847900390625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10142648965120316, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 80995.7109375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010491409339010715, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5227963328361511, "metrics/particle/momentum/all/assigned/pz_bias": -0.0249624140560627, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7025582790374756, "metrics/particle/momentum/all/assigned/samples": 34981.0, "metrics/particle/momentum/all/efficiency": 0.8470446467399597, "metrics/particle/momentum/all/event/energy_bias": -1.3642501831054688, "metrics/particle/momentum/all/event/p3_error_mean": 11.995606422424316, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.178889274597168, "metrics/particle/momentum/all/event/px_bias": 0.38950496912002563, "metrics/particle/momentum/all/event/py_bias": -0.0909905880689621, "metrics/particle/momentum/all/event/pz_abs_mean": 8.791259765625, "metrics/particle/momentum/all/event/pz_bias": -1.5299129486083984, "metrics/particle/momentum/all/event/pz_residual_rms": 16.079120635986328, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0076936352998018265, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.044226594269275665, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.035362228751182556, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004831833764910698, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08103932440280914, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011669328669086099, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12915118038654327, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007694147061556578, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8507977724075317, "metrics/particle/momentum/class_1/accepted/samples": 16282.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0642278641462326, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1687808781862259, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1933472603559494, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1456497460603714, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03459945321083069, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.938321828842163, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0040842522867023945, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2475060522556305, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.015288418158888817, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.649224877357483, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9497753977775574, "metrics/particle/momentum/class_1/event/energy_bias": 1.5561575889587402, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.366462707519531, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.880185604095459, "metrics/particle/momentum/class_1/event/px_bias": 0.19733941555023193, "metrics/particle/momentum/class_1/event/py_bias": -0.13105136156082153, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.985331058502197, "metrics/particle/momentum/class_1/event/pz_bias": -0.16233426332473755, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.754749298095703, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03207935765385628, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1906459778547287, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19208689033985138, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04211614280939102, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23103271424770355, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04196816310286522, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.740799903869629, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.22068709135055542, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.0814595222473145, "metrics/particle/momentum/class_2/accepted/samples": 1382.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7877016067504883, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6575138568878174, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5811.671875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5811.220703125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33985424041748047, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 296693.21875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0736478716135025, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7974863052368164, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.25014159083366394, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.440446376800537, "metrics/particle/momentum/class_2/assigned/samples": 2607.0, "metrics/particle/momentum/class_2/efficiency": 0.4782007038593292, "metrics/particle/momentum/class_2/event/energy_bias": -6.352118015289307, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.76681900024414, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.401761054992676, "metrics/particle/momentum/class_2/event/px_bias": -0.08106698095798492, "metrics/particle/momentum/class_2/event/py_bias": 0.1393403559923172, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.456908226013184, "metrics/particle/momentum/class_2/event/pz_bias": -2.0961098670959473, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.76374340057373, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01720849983394146, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1318923383951187, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1307491660118103, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.032160185277462006, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17212052643299103, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005757052567787468, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16616235673427582, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.012382022105157375, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6188421845436096, "metrics/particle/momentum/class_3/accepted/samples": 12159.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2580729126930237, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3328144550323486, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2041131258010864, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0819742679595947, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13693228363990784, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.98862361907959, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0068670110777020454, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.44291219115257263, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0005771943251602352, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5959017276763916, "metrics/particle/momentum/class_3/assigned/samples": 14708.0, "metrics/particle/momentum/class_3/efficiency": 0.800513505935669, "metrics/particle/momentum/class_3/event/energy_bias": 4.469114303588867, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.341711044311523, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.658159255981445, "metrics/particle/momentum/class_3/event/px_bias": 0.1612558662891388, "metrics/particle/momentum/class_3/event/py_bias": -0.30675190687179565, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.83737850189209, "metrics/particle/momentum/class_3/event/pz_bias": 0.36825448274612427, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.252951622009277, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01144521962851286, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1561100333929062, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10605746507644653, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05545450374484062, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15886849164962769, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003448275849223137, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.37080031633377075, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.02109004743397236, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.184256911277771, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22577238082885742, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6069271564483643, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4488390684127808, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3995087146759033, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14689265191555023, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.915849685668945, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5880207419395447, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.06348639726638794, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0959956645965576, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8192090392112732, "metrics/particle/momentum/class_4/event/energy_bias": -1.1686732769012451, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.965552568435669, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4973976612091064, "metrics/particle/momentum/class_4/event/px_bias": 0.06955546885728836, "metrics/particle/momentum/class_4/event/py_bias": 0.20296135544776917, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0363963842391968, "metrics/particle/momentum/class_4/event/pz_bias": 0.22555184364318848, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.248409748077393, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0055271415039896965, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02332654781639576, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022046342492103577, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005680285859853029, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0607110895216465, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1465844362974167, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.060446448624134064, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6053772568702698, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.014247414655983448, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04184943437576294, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0388190858066082, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006494044791907072, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12516160309314728, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.1755571812391281, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.057650428265333176, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.6501399874687195, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.13126882910728455, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8213436603546143, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6521788239479065, "metrics/particle/momentum/class_5/event/px_bias": 0.042421240359544754, "metrics/particle/momentum/class_5/event/py_bias": 0.004510955885052681, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.38006219267845154, "metrics/particle/momentum/class_5/event/pz_bias": 0.1347246766090393, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6859217882156372, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.7738025188446045, "step_valid_time": 19620.341790676117, "step_total_time": 19622.1155936718, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015725552929597, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06645039664023988, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2939, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9645553002953725, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9632907243526713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9639225975729747, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036709275647328746, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9609963946247132, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9616267628730731, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9645553002953725, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010077257396746, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.071463623306812, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2192, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2186, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9622475856014048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9622475856014048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9622475856014048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03775241439859526, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596136962247586, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9596136962247586, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9596136962247586, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9622475856014048, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0037013452543972, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06931065537508974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1239, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1236, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400606980273141, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9400606980273141, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9400606980273141, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05993930197268589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9377845220030349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400606980273141} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_45000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..7436e0b3216524ab9428aa46e1ac034597804596 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131489200454718, "train": {"Classification_binary": 2.440721035003662, "Classification": 0.016292214393615723, "Regression_pt": 0.11937051266431808, "Regression_eta": 0.009785594418644905, "Regression_sin_phi": 0.006094292737543583, "Regression_cos_phi": 0.005824278574436903, "Regression_energy": 0.12087587267160416, "Total": 7.228687286376953, "Total_unweighted": 2.7189638004638255}, "train_group": {}, "valid": {"Classification_binary": 2.3298139572143555, "Classification": 0.016852409889300663, "Regression_pt": 0.1201564868291219, "Regression_eta": 0.010934567699829737, "Regression_sin_phi": 0.006372788300116857, "Regression_cos_phi": 0.006743917241692543, "Regression_energy": 0.12501696745554605, "Total": 7.337395985921224, "Total_unweighted": 2.6158910946299634, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016607139259576797, "diagnostic/loss/cld_pf/Classification_binary": 2.331693410873413, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006615061312913895, "diagnostic/loss/cld_pf/Regression_energy": 0.1242612674832344, "diagnostic/loss/cld_pf/Regression_eta": 0.010299334302544594, "diagnostic/loss/cld_pf/Regression_pt": 0.12012188136577606, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006045592948794365, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08940472453832626, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18256661295890808, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06228543817996979, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4067108631134033, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.035666849464178085, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17737193405628204, "diagnostic/regression/cld_pf/pt_residual_mean": 0.0628339871764183, "diagnostic/regression/cld_pf/pt_residual_rms": 0.39806175231933594, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9503238797187805, "metrics/particle/class_1/pid_accuracy": 0.9871939420700073, "metrics/particle/class_2/efficiency": 0.47578898072242737, "metrics/particle/class_2/pid_accuracy": 0.7957375645637512, "metrics/particle/class_3/efficiency": 0.7905898094177246, "metrics/particle/class_3/pid_accuracy": 0.9865184426307678, "metrics/particle/class_4/efficiency": 0.82341068983078, "metrics/particle/class_4/pid_accuracy": 0.4325980246067047, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.705765426158905, "metrics/particle/count/bias_mean": -0.16862745583057404, "metrics/particle/count/mae": 3.0, "metrics/particle/count/prediction_mean": 70.396728515625, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.045429863035678864, "metrics/particle/event/energy_response_mean": 1.0010058879852295, "metrics/particle/event/met_abs_error": 4.635030746459961, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03754151239991188, "metrics/particle/event/scalar_pt_response_mean": 1.0040192604064941, "metrics/particle/event/vector_pt_closure": 0.04128164052963257, "metrics/particle/matched/delta_eta_abs_mean": 0.007436415646225214, "metrics/particle/matched/delta_phi_abs_mean": 0.008431891910731792, "metrics/particle/matched/delta_r_mean": 0.012651284225285053, "metrics/particle/matched/energy_relative_abs_mean": 0.0858924388885498, "metrics/particle/matched/pid_accuracy": 0.9716218709945679, "metrics/particle/matched/pt_relative_abs_mean": 0.08056392520666122, "metrics/particle/matching/duplicate_fraction": 0.02757480926811695, "metrics/particle/matching/efficiency": 0.8433844447135925, "metrics/particle/matching/f1": 0.8443933129310608, "metrics/particle/matching/purity": 0.8454046845436096, "metrics/particle/matching/target_coverage_dr0p05": 0.8328810334205627, "metrics/particle/matching/target_coverage_dr0p10": 0.8766359686851501, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7436020970344543, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8768396973609924, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01448612380772829, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0823320597410202, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07847461104393005, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01901848427951336, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13095039129257202, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0044052861630916595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.34261226654052734, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.003330081235617399, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.431797981262207, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8399.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2258019745349884, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8078316450119019, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7410205602645874, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6406853795051575, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10914065688848495, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.514570236206055, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012363194487988949, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6562360525131226, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.018085438758134842, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5089516639709473, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9868.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8285489082336426, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.034658998250961304, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.0792951583862305, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.621412754058838, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11984901130199432, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.20749586820602417, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.303376197814941, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4163799285888672, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.307486534118652, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022873414680361748, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09155531227588654, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08976205438375473, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014020023867487907, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1401359885931015, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022195560857653618, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9533287882804871, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1783270388841629, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.014463424682617, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1667.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7197852730751038, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7059377431869507, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6089.263671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6089.1015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2549552917480469, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 308729.53125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07267780601978302, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6665308475494385, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0038843611255288124, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.780721187591553, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2573.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6064023375511169, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.43436652421951294, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.123693943023682, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8794858455657959, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10993371903896332, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07841625809669495, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.993868350982666, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2818383276462555, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.151224136352539, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010913817211985588, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08787882328033447, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0818195790052414, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015220541507005692, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13599054515361786, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00034869241062551737, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1030590608716011, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.001118568587116897, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4631723463535309, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20075.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1263398975133896, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6104968786239624, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4513227343559265, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.34743088483810425, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08408141136169434, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.080292701721191, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021775842178612947, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2198282927274704, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.004188493359833956, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2331430912017822, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22502.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8780562281608582, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.24575874209403992, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.527944564819336, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.689441680908203, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.21489548683166504, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.43786633014678955, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.4566407203674316, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.03753836452960968, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.486428260803223, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012570710852742195, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08653650432825089, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08132674545049667, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016212468966841698, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1348399966955185, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026873694732785225, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21683786809444427, "metrics/particle/momentum/all/accepted/pz_bias": -0.011535643599927425, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1030480861663818, "metrics/particle/momentum/all/accepted/samples": 30141.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19812613725662231, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.746886670589447, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 448.8779602050781, "metrics/particle/momentum/all/assigned/pt_relative_bias": 448.77081298828125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1037403792142868, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 83775.6796875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010245256125926971, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5232320427894592, "metrics/particle/momentum/all/assigned/pz_bias": -0.002124121179804206, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6860201358795166, "metrics/particle/momentum/all/assigned/samples": 34943.0, "metrics/particle/momentum/all/efficiency": 0.843128502368927, "metrics/particle/momentum/all/event/energy_bias": -0.7147842645645142, "metrics/particle/momentum/all/event/p3_error_mean": 12.540496826171875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.459969520568848, "metrics/particle/momentum/all/event/px_bias": -0.014887260273098946, "metrics/particle/momentum/all/event/py_bias": -0.5669459700584412, "metrics/particle/momentum/all/event/pz_abs_mean": 9.222489356994629, "metrics/particle/momentum/all/event/pz_bias": -0.09700322151184082, "metrics/particle/momentum/all/event/pz_residual_rms": 16.82616424560547, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007496098056435585, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04322989657521248, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03522709384560585, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0052947974763810635, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08076704293489456, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010451890993863344, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12494829297065735, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006765473168343306, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7983497977256775, "metrics/particle/momentum/class_1/accepted/samples": 16265.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06466034799814224, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1622389554977417, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2009231001138687, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15352889895439148, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03588307276368141, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.1244702339172363, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00373417348600924, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24328815937042236, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0022457088343799114, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7170159816741943, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9487837553024292, "metrics/particle/momentum/class_1/event/energy_bias": 1.3785731792449951, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.329285621643066, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.826420783996582, "metrics/particle/momentum/class_1/event/px_bias": 0.3215915560722351, "metrics/particle/momentum/class_1/event/py_bias": -0.2553573548793793, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.9180498123168945, "metrics/particle/momentum/class_1/event/pz_bias": 0.04285536706447601, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.239688873291016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03160981088876724, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1883690357208252, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18981380760669708, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.038043126463890076, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22973676025867462, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04123711213469505, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.704608678817749, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1631620228290558, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.925215244293213, "metrics/particle/momentum/class_2/accepted/samples": 1358.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7914174795150757, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6461291313171387, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5998.84375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5998.39111328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.350057452917099, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 306474.71875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06932210177183151, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.713075876235962, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.17651236057281494, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.897148609161377, "metrics/particle/momentum/class_2/assigned/samples": 2611.0, "metrics/particle/momentum/class_2/efficiency": 0.46989619731903076, "metrics/particle/momentum/class_2/event/energy_bias": -6.034623146057129, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.50638198852539, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.289791107177734, "metrics/particle/momentum/class_2/event/px_bias": -0.07263778895139694, "metrics/particle/momentum/class_2/event/py_bias": -0.36040768027305603, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.199631690979004, "metrics/particle/momentum/class_2/event/pz_bias": -1.5127893686294556, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.782000541687012, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017394930124282837, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13276667892932892, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13145898282527924, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03597276285290718, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1731749176979065, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005802868399769068, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.17057570815086365, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0025025391951203346, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6372138857841492, "metrics/particle/momentum/class_3/accepted/samples": 12063.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24923235177993774, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3731589317321777, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2162457704544067, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.098601222038269, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13875630497932434, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.957115173339844, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007227601483464241, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.457828164100647, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.025870107114315033, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.696359157562256, "metrics/particle/momentum/class_3/assigned/samples": 14666.0, "metrics/particle/momentum/class_3/efficiency": 0.7941931486129761, "metrics/particle/momentum/class_3/event/energy_bias": 5.071785926818848, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.387883186340332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.641023635864258, "metrics/particle/momentum/class_3/event/px_bias": -0.2886689007282257, "metrics/particle/momentum/class_3/event/py_bias": -0.13726960122585297, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.925979137420654, "metrics/particle/momentum/class_3/event/pz_bias": 0.8394430875778198, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.016799926757812, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011613275855779648, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1550590693950653, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11040419340133667, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06712312996387482, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1678125411272049, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0034722222480922937, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.38716021180152893, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030001355335116386, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2452385425567627, "metrics/particle/momentum/class_4/accepted/samples": 288.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.25997352600097656, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.195246696472168, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.471367359161377, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.413536548614502, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15819208323955536, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.23725700378418, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016949152573943138, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7909685373306274, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.09280326217412949, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.3542652130126953, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8135592937469482, "metrics/particle/momentum/class_4/event/energy_bias": -1.2775228023529053, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7670543193817139, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3488324880599976, "metrics/particle/momentum/class_4/event/px_bias": -0.11872442811727524, "metrics/particle/momentum/class_4/event/py_bias": 0.19177491962909698, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9158279895782471, "metrics/particle/momentum/class_4/event/pz_bias": 0.4359129071235657, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.3548996448516846, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0051736352033913136, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.018778929486870766, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.0176498144865036, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005579774733632803, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04697198420763016, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11623068153858185, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.032732658088207245, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4802587926387787, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.00719434954226017, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.040109384804964066, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03733474388718605, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.009929558262228966, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12203110009431839, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20364288985729218, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.07443054020404816, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9743273258209229, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.14700284600257874, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7956894636154175, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6338657140731812, "metrics/particle/momentum/class_5/event/px_bias": 0.14355231821537018, "metrics/particle/momentum/class_5/event/py_bias": -0.00568627193570137, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3528052568435669, "metrics/particle/momentum/class_5/event/pz_bias": 0.09757474064826965, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.624172329902649, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.686079740524292, "step_valid_time": 20068.015287160873, "step_total_time": 20069.701367139816, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019495721668017, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06594350740496735, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2944, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9661962586150312, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9674663161353927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9668308702791462, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.032533683864607296, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9622579586478504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9635228393033191, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9628899835796387, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9661962586150312, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031053586331182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07071110314162266, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2177, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9591747146619842, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9608619173262972, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9600175746924429, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03913808267370272, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9556628621597892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9573438874230431, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9565026362038664, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9591747146619842, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052289080188586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07418403861239753, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9415781487101669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9387291981845688, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9401515151515152, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06127080181543117, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355083459787557, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.93267776096823, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9340909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9415781487101669} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_46000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..7b10e0f22bb019141b019f0f804726c06fd405d0 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.1788556271314894, "train": {"Classification_binary": 2.3083906173706055, "Classification": 0.016748767346143723, "Regression_pt": 0.1172131672501564, "Regression_eta": 0.0179370678961277, "Regression_sin_phi": 0.006903755012899637, "Regression_cos_phi": 0.007649495266377926, "Regression_energy": 0.12573277950286865, "Total": 7.725263595581055, "Total_unweighted": 2.6005756496451795}, "train_group": {}, "valid": {"Classification_binary": 2.321675936381022, "Classification": 0.01685488224029541, "Regression_pt": 0.11967390775680542, "Regression_eta": 0.01083286851644516, "Regression_sin_phi": 0.006336771572629611, "Regression_cos_phi": 0.006719596063097318, "Regression_energy": 0.1243910292784373, "Total": 7.307738622029622, "Total_unweighted": 2.606484991808732, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016624700278043747, "diagnostic/loss/cld_pf/Classification_binary": 2.3262386322021484, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006480744108557701, "diagnostic/loss/cld_pf/Regression_energy": 0.12282323837280273, "diagnostic/loss/cld_pf/Regression_eta": 0.010124566964805126, "diagnostic/loss/cld_pf/Regression_pt": 0.1189655140042305, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005960003472864628, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08064869791269302, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.17989489436149597, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05352942273020744, "diagnostic/regression/cld_pf/energy_residual_rms": 0.401652067899704, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.02997537888586521, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1758679747581482, "diagnostic/regression/cld_pf/pt_residual_mean": 0.05714251473546028, "diagnostic/regression/cld_pf/pt_residual_rms": 0.395243763923645, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9507647156715393, "metrics/particle/class_1/pid_accuracy": 0.9876635074615479, "metrics/particle/class_2/efficiency": 0.4741939306259155, "metrics/particle/class_2/pid_accuracy": 0.7912061214447021, "metrics/particle/class_3/efficiency": 0.7938423156738281, "metrics/particle/class_3/pid_accuracy": 0.987514853477478, "metrics/particle/class_4/efficiency": 0.8375378251075745, "metrics/particle/class_4/pid_accuracy": 0.4457831382751465, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6926147937774658, "metrics/particle/count/bias_mean": -0.5862745046615601, "metrics/particle/count/mae": 3.029411792755127, "metrics/particle/count/prediction_mean": 69.97908782958984, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04644309729337692, "metrics/particle/event/energy_response_mean": 0.9927214980125427, "metrics/particle/event/met_abs_error": 4.413580894470215, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036720700562000275, "metrics/particle/event/scalar_pt_response_mean": 0.99867844581604, "metrics/particle/event/vector_pt_closure": 0.03976213559508324, "metrics/particle/matched/delta_eta_abs_mean": 0.007305205799639225, "metrics/particle/matched/delta_phi_abs_mean": 0.00812563020735979, "metrics/particle/matched/delta_r_mean": 0.012314100749790668, "metrics/particle/matched/energy_relative_abs_mean": 0.08472803235054016, "metrics/particle/matched/pid_accuracy": 0.9720909595489502, "metrics/particle/matched/pt_relative_abs_mean": 0.08022116869688034, "metrics/particle/matching/duplicate_fraction": 0.026908133178949356, "metrics/particle/matching/efficiency": 0.8449497818946838, "metrics/particle/matching/f1": 0.8484744429588318, "metrics/particle/matching/purity": 0.8520286083221436, "metrics/particle/matching/target_coverage_dr0p05": 0.8326680064201355, "metrics/particle/matching/target_coverage_dr0p10": 0.8752651214599609, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7462047934532166, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8777844905853271, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014008216559886932, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08074447512626648, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07773993164300919, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01541041862219572, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12900689244270325, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00441685551777482, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.34050968289375305, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0024034082889556885, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4192206859588623, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8377.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22896255552768707, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8044154047966003, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7693727612495422, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.668708860874176, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10848528146743774, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.251001358032227, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012529285624623299, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6497049927711487, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0027014927472919226, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5093677043914795, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9817.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.826378583908081, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5437425374984741, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.753942489624023, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5257272720336914, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.013752371072769165, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.11279009282588959, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.977511405944824, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.6656714677810669, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.778945922851562, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02271280065178871, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09214267134666443, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09033858776092529, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01526784710586071, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14029350876808167, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02274087443947792, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9596503973007202, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18401633203029633, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1150100231170654, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1671.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6992726922035217, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7655417919158936, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17649.439453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17649.275390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2453867346048355, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 890552.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0749901831150055, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.634226083755493, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13998766243457794, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.797276020050049, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2547.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6078574061393738, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.2007153034210205, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.49090576171875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.885420560836792, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.049900397658348083, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06567666679620743, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.367610454559326, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.01840841770172119, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.337239265441895, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010509115643799305, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08704332262277603, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08169286698102951, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011201173067092896, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13574600219726562, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00024829915491864085, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10382480174303055, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0021407597232609987, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.45668527483940125, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20137.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12828879058361053, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.590651273727417, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4110786020755768, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3041485548019409, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08076580613851547, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.0176615715026855, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021816561929881573, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2237236350774765, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007442089729011059, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1704182624816895, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22460.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8807680606842041, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.7639564871788025, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.589338779449463, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.7268452644348145, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.23558685183525085, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.008388534188270569, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.5325584411621094, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2497822791337967, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.462785243988037, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012155772186815739, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08557754755020142, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08107444643974304, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012594454921782017, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13416898250579834, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026503230910748243, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21688738465309143, "metrics/particle/momentum/all/accepted/pz_bias": -0.009425747208297253, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.111430287361145, "metrics/particle/momentum/all/accepted/samples": 30185.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1984303593635559, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7368426322937012, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1291.34814453125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1291.2386474609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10062026232481003, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 240843.421875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010423845611512661, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5201115012168884, "metrics/particle/momentum/all/assigned/pz_bias": -0.0062003168277442455, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6655807495117188, "metrics/particle/momentum/all/assigned/samples": 34824.0, "metrics/particle/momentum/all/efficiency": 0.844359278678894, "metrics/particle/momentum/all/event/energy_bias": -2.5084145069122314, "metrics/particle/momentum/all/event/p3_error_mean": 12.329710960388184, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.215390205383301, "metrics/particle/momentum/all/event/px_bias": 0.19943882524967194, "metrics/particle/momentum/all/event/py_bias": -0.038724884390830994, "metrics/particle/momentum/all/event/pz_abs_mean": 9.174044609069824, "metrics/particle/momentum/all/event/pz_bias": -0.4342975616455078, "metrics/particle/momentum/all/event/pz_residual_rms": 17.10147476196289, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007513217628002167, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04248575121164322, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03549909591674805, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0037243012338876724, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08128226548433304, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011670761741697788, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1258743703365326, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006108745466917753, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7670549750328064, "metrics/particle/momentum/class_1/accepted/samples": 16280.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06482449173927307, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18865592777729034, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2148522585630417, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.165291890501976, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03524331748485565, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.261040449142456, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004084490705281496, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24383606016635895, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.012731365859508514, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6641124486923218, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9496587514877319, "metrics/particle/momentum/class_1/event/energy_bias": 1.1794414520263672, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.803070068359375, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.384936332702637, "metrics/particle/momentum/class_1/event/px_bias": 0.452747106552124, "metrics/particle/momentum/class_1/event/py_bias": 0.024302855134010315, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.701974391937256, "metrics/particle/momentum/class_1/event/pz_bias": -0.5461671352386475, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.6775484085083, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03117755427956581, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18934285640716553, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19130834937095642, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04109920188784599, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23076297342777252, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.038235295563936234, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7098923921585083, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.127948597073555, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.006809711456299, "metrics/particle/momentum/class_2/accepted/samples": 1360.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7794149518013, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6797519326210022, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17448.419921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17447.9765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34510868787765503, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 885525.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07065217196941376, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.774333953857422, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.11208158731460571, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.385659217834473, "metrics/particle/momentum/class_2/assigned/samples": 2576.0, "metrics/particle/momentum/class_2/efficiency": 0.47058823704719543, "metrics/particle/momentum/class_2/event/energy_bias": -7.2080559730529785, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.656684875488281, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.4059648513793945, "metrics/particle/momentum/class_2/event/px_bias": 0.10308907181024551, "metrics/particle/momentum/class_2/event/py_bias": -0.1091640293598175, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.254589080810547, "metrics/particle/momentum/class_2/event/pz_bias": -1.220038890838623, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.81718635559082, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01637006737291813, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13126999139785767, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1302986890077591, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02963591180741787, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17151197791099548, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000579278392251581, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16802813112735748, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.002051772316917777, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.664193332195282, "metrics/particle/momentum/class_3/accepted/samples": 12084.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25362181663513184, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3575466871261597, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2811278104782104, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1580003499984741, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1343345046043396, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.341489791870117, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007200164720416069, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.44594520330429077, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.013589011505246162, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5027377605438232, "metrics/particle/momentum/class_3/assigned/samples": 14583.0, "metrics/particle/momentum/class_3/efficiency": 0.795575737953186, "metrics/particle/momentum/class_3/event/energy_bias": 4.54196834564209, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.431573867797852, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.575849533081055, "metrics/particle/momentum/class_3/event/px_bias": -0.39738649129867554, "metrics/particle/momentum/class_3/event/py_bias": -0.13130080699920654, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.003506183624268, "metrics/particle/momentum/class_3/event/pz_bias": 0.8191672563552856, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.165401458740234, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01196714024990797, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15014046430587769, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10524296015501022, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05746322497725487, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15749147534370422, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006802720949053764, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.41204333305358887, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04084935411810875, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3121528625488281, "metrics/particle/momentum/class_4/accepted/samples": 294.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.251211553812027, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.461883783340454, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.1537749767303467, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1026307344436646, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1355932205915451, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.159440517425537, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7027412056922913, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2925141453742981, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.114839553833008, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8305084705352783, "metrics/particle/momentum/class_4/event/energy_bias": -1.130676507949829, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.685532569885254, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3147926330566406, "metrics/particle/momentum/class_4/event/px_bias": -0.07080334424972534, "metrics/particle/momentum/class_4/event/py_bias": 0.19247443974018097, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8398571610450745, "metrics/particle/momentum/class_4/event/pz_bias": 0.3747645914554596, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.2832045555114746, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005216255318373442, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021413138136267662, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021897444501519203, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0024694253224879503, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06138228252530098, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12254007160663605, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.010348591953516006, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4699980616569519, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.022589193657040596, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04029864817857742, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.038334690034389496, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.012904508039355278, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11488725990056992, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20129884779453278, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.062000472098588943, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.953289270401001, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.1089075580239296, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8020192384719849, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6480461359024048, "metrics/particle/momentum/class_5/event/px_bias": 0.11179251223802567, "metrics/particle/momentum/class_5/event/py_bias": -0.015037361532449722, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3523916006088257, "metrics/particle/momentum/class_5/event/pz_bias": 0.1379765421152115, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.5856274366378784, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.789710283279419, "step_valid_time": 20515.773169994354, "step_total_time": 20517.562880516052, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998763230855745, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06495053379641424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625861503117821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9689461513049223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9657556799473165, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.031053848695077636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596324253363965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.965972910472415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9627922291735265, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625861503117821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995765812563301, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07258053748581128, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2276, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9591747146619842, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9600175746924429, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9595959595959596, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03998242530755712, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552238805970149, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9560632688927944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556433904259991, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9591747146619842, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026168135450222, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07285968438896362, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1246, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9453717754172989, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9446550416982562, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9450132726583238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05534495830174375, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9415781487101669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9408642911296436, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9412210845657945, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9453717754172989} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_47000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..c95e7daa9f24a4429d4f6bd1f649fb0b78d8f9d0 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226222053808261, "train": {"Classification_binary": 2.3535680770874023, "Classification": 0.018843820318579674, "Regression_pt": 0.1054762452840805, "Regression_eta": 0.008901655673980713, "Regression_sin_phi": 0.006186498329043388, "Regression_cos_phi": 0.006129270885139704, "Regression_energy": 0.10713542252779007, "Total": 6.926584243774414, "Total_unweighted": 2.6062409901060164}, "train_group": {}, "valid": {"Classification_binary": 2.320598761240641, "Classification": 0.01683620239297549, "Regression_pt": 0.11957227190335591, "Regression_eta": 0.010938778519630432, "Regression_sin_phi": 0.006331849843263626, "Regression_cos_phi": 0.006695890178283055, "Regression_energy": 0.12429357568422954, "Total": 7.306734085083008, "Total_unweighted": 2.605267329762379, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016645263880491257, "diagnostic/loss/cld_pf/Classification_binary": 2.3241567611694336, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006476587615907192, "diagnostic/loss/cld_pf/Regression_energy": 0.12298476696014404, "diagnostic/loss/cld_pf/Regression_eta": 0.010114563629031181, "diagnostic/loss/cld_pf/Regression_pt": 0.11911451071500778, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005920238792896271, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09444928169250488, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.183388352394104, "diagnostic/regression/cld_pf/energy_residual_mean": 0.067330002784729, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4105144441127777, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04039059579372406, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17776484787464142, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06755772978067398, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4017980694770813, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9502472281455994, "metrics/particle/class_1/pid_accuracy": 0.9862651824951172, "metrics/particle/class_2/efficiency": 0.47829553484916687, "metrics/particle/class_2/pid_accuracy": 0.7975226044654846, "metrics/particle/class_3/efficiency": 0.7901942729949951, "metrics/particle/class_3/pid_accuracy": 0.9859276413917542, "metrics/particle/class_4/efficiency": 0.8224016427993774, "metrics/particle/class_4/pid_accuracy": 0.46871164441108704, "metrics/particle/class_5/efficiency": 0.9786407947540283, "metrics/particle/class_5/pid_accuracy": 0.7083333134651184, "metrics/particle/count/bias_mean": -0.22026143968105316, "metrics/particle/count/mae": 3.020261526107788, "metrics/particle/count/prediction_mean": 70.34510040283203, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04745187237858772, "metrics/particle/event/energy_response_mean": 1.0061955451965332, "metrics/particle/event/met_abs_error": 4.63388729095459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038684140890836716, "metrics/particle/event/scalar_pt_response_mean": 1.011017918586731, "metrics/particle/event/vector_pt_closure": 0.04137057438492775, "metrics/particle/matched/delta_eta_abs_mean": 0.007447374053299427, "metrics/particle/matched/delta_phi_abs_mean": 0.008192164823412895, "metrics/particle/matched/delta_r_mean": 0.012456842698156834, "metrics/particle/matched/energy_relative_abs_mean": 0.08601386845111847, "metrics/particle/matched/pid_accuracy": 0.9712594151496887, "metrics/particle/matched/pt_relative_abs_mean": 0.08055906742811203, "metrics/particle/matching/duplicate_fraction": 0.02922102063894272, "metrics/particle/matching/efficiency": 0.8433844447135925, "metrics/particle/matching/f1": 0.8447027206420898, "metrics/particle/matching/purity": 0.8460251688957214, "metrics/particle/matching/target_coverage_dr0p05": 0.8339924812316895, "metrics/particle/matching/target_coverage_dr0p10": 0.8772101998329163, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7442874908447266, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8761172890663147, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014059865847229958, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08173976093530655, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07763209939002991, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018182670697569847, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12875965237617493, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004777831025421619, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3496447801589966, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.003921179100871086, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.496901512145996, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8372.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23118112981319427, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9244630932807922, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8531883955001831, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7562775015830994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11335498094558716, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.051508903503418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013294093310832977, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6821184754371643, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.01303422823548317, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.637634515762329, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9854.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8258853554725647, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5517614483833313, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.949357032775879, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5860705375671387, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.06962858140468597, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.10150140523910522, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.173192024230957, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.36999812722206116, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.749649047851562, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022782867774367332, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09126085788011551, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08928649872541428, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01804022118449211, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13931386172771454, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020395921543240547, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.898268461227417, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16008645296096802, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.875615358352661, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1667.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6800025105476379, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.49028480052948, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 17542.287109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 17542.1328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2437007874250412, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 883975.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07165354490280151, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.739187002182007, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0027929486241191626, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.753691673278809, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2540.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6064023375511169, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.8433124423027039, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.11501407623291, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8628343343734741, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0806637853384018, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05368687957525253, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.991481781005859, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.7281644344329834, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.58908748626709, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01075027696788311, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08808290958404541, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08186335116624832, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018860215321183205, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1356375515460968, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0002494014333933592, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10513639450073242, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005747270304709673, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4653392732143402, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20048.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12817959487438202, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5872891545295715, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4179301857948303, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.31868383288383484, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08499577641487122, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.921273946762085, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002177100395783782, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22602616250514984, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007086662575602531, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1508921384811401, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22507.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8768752813339233, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.4138600826263428, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.57839298248291, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.745267868041992, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.0038728900253772736, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.18449124693870544, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.4489588737487793, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.29787299036979675, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.382458686828613, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012337880209088326, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08649394661188126, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08109725266695023, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018626248463988304, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13396841287612915, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026257187128067017, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21711748838424683, "metrics/particle/momentum/all/accepted/pz_bias": -0.0039490386843681335, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1072263717651367, "metrics/particle/momentum/all/accepted/samples": 30087.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1974213719367981, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7482049465179443, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1277.1903076171875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1277.0877685546875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10455287992954254, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 238472.40625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010372195392847061, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5377009510993958, "metrics/particle/momentum/all/assigned/pz_bias": 0.008453422226011753, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8974430561065674, "metrics/particle/momentum/all/assigned/samples": 34901.0, "metrics/particle/momentum/all/efficiency": 0.8416179418563843, "metrics/particle/momentum/all/event/energy_bias": 1.1223089694976807, "metrics/particle/momentum/all/event/p3_error_mean": 12.678990364074707, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.494970798492432, "metrics/particle/momentum/all/event/px_bias": -0.15416525304317474, "metrics/particle/momentum/all/event/py_bias": -0.3396795392036438, "metrics/particle/momentum/all/event/pz_abs_mean": 9.382698059082031, "metrics/particle/momentum/all/event/pz_bias": 0.6560393571853638, "metrics/particle/momentum/all/event/pz_residual_rms": 18.351896286010742, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007524092681705952, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04391472414135933, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.035740602761507034, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0069323135539889336, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08133015036582947, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0008611675002612174, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12338840216398239, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.001400940353050828, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.745920717716217, "metrics/particle/momentum/class_1/accepted/samples": 16257.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06621824204921722, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21585851907730103, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22898295521736145, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18302972614765167, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03658536449074745, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.2485949993133545, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00385109125636518, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.25971004366874695, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.014726236462593079, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1811304092407227, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9483171105384827, "metrics/particle/momentum/class_1/event/energy_bias": 1.5491127967834473, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.55899715423584, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.855451583862305, "metrics/particle/momentum/class_1/event/px_bias": 0.023063763976097107, "metrics/particle/momentum/class_1/event/py_bias": -0.15559041500091553, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.313640117645264, "metrics/particle/momentum/class_1/event/pz_bias": 0.6434340476989746, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.711418151855469, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031133214011788368, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1889929324388504, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.190347358584404, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03921937197446823, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2306509166955948, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04245973750948906, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7555615901947021, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11640536040067673, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.129301071166992, "metrics/particle/momentum/class_2/accepted/samples": 1366.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7702455520629883, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7188921570777893, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 17077.068359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 17076.642578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34457647800445557, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 872207.9375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07282483577728271, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8805904388427734, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.015079918317496777, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.713541030883789, "metrics/particle/momentum/class_2/assigned/samples": 2609.0, "metrics/particle/momentum/class_2/efficiency": 0.47266435623168945, "metrics/particle/momentum/class_2/event/energy_bias": -4.937088489532471, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.470565795898438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.349921226501465, "metrics/particle/momentum/class_2/event/px_bias": 0.02566985785961151, "metrics/particle/momentum/class_2/event/py_bias": 0.13478544354438782, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.14660930633545, "metrics/particle/momentum/class_2/event/pz_bias": -1.2628014087677002, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.548019409179688, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016828790307044983, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13190828263759613, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13040313124656677, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0402044951915741, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17138126492500305, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005827020504511893, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16674812138080597, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.005674131214618683, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5916374921798706, "metrics/particle/momentum/class_3/accepted/samples": 12013.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.249685138463974, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3462929725646973, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1461443901062012, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0330331325531006, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14069870114326477, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.119489669799805, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006973405368626118, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.44607746601104736, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.004085817374289036, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4878673553466797, "metrics/particle/momentum/class_3/assigned/samples": 14627.0, "metrics/particle/momentum/class_3/efficiency": 0.7909013032913208, "metrics/particle/momentum/class_3/event/energy_bias": 5.221190452575684, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.276734352111816, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.587695598602295, "metrics/particle/momentum/class_3/event/px_bias": -0.21549269556999207, "metrics/particle/momentum/class_3/event/py_bias": -0.44251418113708496, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.841639518737793, "metrics/particle/momentum/class_3/event/pz_bias": 0.8325676918029785, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.909191131591797, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011791499331593513, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1491025984287262, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10328957438468933, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06120391562581062, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15229572355747223, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3732456564903259, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.03260345384478569, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0403903722763062, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2610999643802643, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.368817090988159, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.290849208831787, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.23279869556427, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16949152946472168, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.428787231445312, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6770684719085693, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0921613946557045, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.9934635162353516, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8022598624229431, "metrics/particle/momentum/class_4/event/energy_bias": -0.958741307258606, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8903851509094238, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4354441165924072, "metrics/particle/momentum/class_4/event/px_bias": -0.07583101838827133, "metrics/particle/momentum/class_4/event/py_bias": 0.155117928981781, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9901078939437866, "metrics/particle/momentum/class_4/event/pz_bias": 0.3071311414241791, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.1226677894592285, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005087481811642647, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019748281687498093, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.018303310498595238, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005536103621125221, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04614726081490517, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11523782461881638, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.024347949773073196, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4616391658782959, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.006996476091444492, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.042420826852321625, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03888434171676636, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.018153050914406776, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12507694959640503, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2050454318523407, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.060058947652578354, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9951038956642151, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.24783574044704437, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8570572137832642, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6827888488769531, "metrics/particle/momentum/class_5/event/px_bias": 0.08842484652996063, "metrics/particle/momentum/class_5/event/py_bias": -0.03147828206419945, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3818668723106384, "metrics/particle/momentum/class_5/event/pz_bias": 0.1357080042362213, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.692421793937683, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.7079358100891113, "step_valid_time": 20963.633895158768, "step_total_time": 20965.341831207275, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0049541030678464, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06859756581163767, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2937, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2926, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9638989169675091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.964215364412344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9640571147218119, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03578463558765594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602888086642599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9606040709126723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9604464139176103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9638989169675091, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0062614258886058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0730555916389446, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2280, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2191, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2187, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9618086040386303, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9609649122807018, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.961386573058359, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03903508771929825, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9600526777875329, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9592105263157895, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9596314172882844, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9618086040386303, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0098297192389412, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07489281242560608, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05690440060698027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9393019726858877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9430955993930197} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_48000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..00bb64c2525d95c70d27f2e95d922242d00c1feb --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.2735884804850324, "train": {"Classification_binary": 2.281405448913574, "Classification": 0.017464371398091316, "Regression_pt": 0.1058790385723114, "Regression_eta": 0.009736819192767143, "Regression_sin_phi": 0.006098134908825159, "Regression_cos_phi": 0.006305933929979801, "Regression_energy": 0.1083512231707573, "Total": 6.856485843658447, "Total_unweighted": 2.5352409700863063}, "train_group": {}, "valid": {"Classification_binary": 2.3221398989359536, "Classification": 0.016779154539108276, "Regression_pt": 0.1193595826625824, "Regression_eta": 0.010751431186993917, "Regression_sin_phi": 0.0063103797535101576, "Regression_cos_phi": 0.006678116197387378, "Regression_energy": 0.124016672372818, "Total": 7.288227081298828, "Total_unweighted": 2.6060352356483536, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016617711633443832, "diagnostic/loss/cld_pf/Classification_binary": 2.326976776123047, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006493077147752047, "diagnostic/loss/cld_pf/Regression_energy": 0.12277945131063461, "diagnostic/loss/cld_pf/Regression_eta": 0.00990305095911026, "diagnostic/loss/cld_pf/Regression_pt": 0.11883921176195145, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005895424634218216, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08810566365718842, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.18167252838611603, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06098638102412224, "diagnostic/regression/cld_pf/energy_residual_rms": 0.405378133058548, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03712408244609833, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17701011896133423, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06429122388362885, "diagnostic/regression/cld_pf/pt_residual_rms": 0.3981740176677704, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.950707197189331, "metrics/particle/class_1/pid_accuracy": 0.9876224398612976, "metrics/particle/class_2/efficiency": 0.47134557366371155, "metrics/particle/class_2/pid_accuracy": 0.7872854471206665, "metrics/particle/class_3/efficiency": 0.7965673208236694, "metrics/particle/class_3/pid_accuracy": 0.9871437549591064, "metrics/particle/class_4/efficiency": 0.8264379501342773, "metrics/particle/class_4/pid_accuracy": 0.4383394420146942, "metrics/particle/class_5/efficiency": 0.9728155136108398, "metrics/particle/class_5/pid_accuracy": 0.6766467094421387, "metrics/particle/count/bias_mean": -0.16601307690143585, "metrics/particle/count/mae": 2.956862688064575, "metrics/particle/count/prediction_mean": 70.39934539794922, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04367152228951454, "metrics/particle/event/energy_response_mean": 0.9960313439369202, "metrics/particle/event/met_abs_error": 4.554033279418945, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03526567295193672, "metrics/particle/event/scalar_pt_response_mean": 1.0029606819152832, "metrics/particle/event/vector_pt_closure": 0.04038979485630989, "metrics/particle/matched/delta_eta_abs_mean": 0.007298086769878864, "metrics/particle/matched/delta_phi_abs_mean": 0.008102970197796822, "metrics/particle/matched/delta_r_mean": 0.012281402014195919, "metrics/particle/matched/energy_relative_abs_mean": 0.08597290515899658, "metrics/particle/matched/pid_accuracy": 0.9717227220535278, "metrics/particle/matched/pt_relative_abs_mean": 0.08102129399776459, "metrics/particle/matching/duplicate_fraction": 0.02749951183795929, "metrics/particle/matching/efficiency": 0.8457370400428772, "metrics/particle/matching/f1": 0.846733033657074, "metrics/particle/matching/purity": 0.8477314114570618, "metrics/particle/matching/target_coverage_dr0p05": 0.8349372744560242, "metrics/particle/matching/target_coverage_dr0p10": 0.8780993819236755, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7453341484069824, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8786088228225708, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.013989799655973911, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08256682753562927, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07928425073623657, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020448995754122734, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13132479786872864, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00451520923525095, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.34406810998916626, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0122177479788661, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.462491750717163, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8416.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22970806062221527, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8236132264137268, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8053969144821167, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7058222889900208, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10852399468421936, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.637689590454102, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012654383666813374, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.646624743938446, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.00975771900266409, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4738430976867676, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9878.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8302258849143982, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3680565357208252, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.6666669845581055, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.494691848754883, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.021308260038495064, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.07231638580560684, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.912662506103516, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.02529364824295044, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.366931915283203, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021845974028110504, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09152714163064957, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09009000658988953, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017733776941895485, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13957011699676514, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019207682460546494, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9170049428939819, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20437508821487427, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0700204372406006, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1666.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.687044084072113, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7576045989990234, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5450.60107421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5450.44970703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24931827187538147, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 275986.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07128944247961044, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.560443162918091, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.11291282624006271, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.423503398895264, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2567.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6060385704040527, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.9503182172775269, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.941778182983398, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8237866759300232, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04723283648490906, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06559363752603531, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.825281620025635, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.02525845170021057, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.49416732788086, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010542558506131172, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0886499211192131, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08262299746274948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016722247004508972, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13709157705307007, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00024824985302984715, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10354913771152496, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005767118185758591, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.45475876331329346, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20141.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1287337690591812, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5738725662231445, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4116404354572296, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3085135519504547, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08242294937372208, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.1556396484375, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0019984012469649315, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21916235983371735, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013652161695063114, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1623413562774658, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22518.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8809430003166199, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.025793949142098427, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.532816410064697, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.773799419403076, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3141043782234192, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.14969520270824432, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.300539493560791, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3647163212299347, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 3.8170526027679443, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012125573121011257, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0871146097779274, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08210489153862, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01781577058136463, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13564980030059814, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002481553703546524, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2153654545545578, "metrics/particle/momentum/all/accepted/pz_bias": -0.004020407795906067, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1193567514419556, "metrics/particle/momentum/all/accepted/samples": 30223.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19825316965579987, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7313411831855774, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 400.6783752441406, "metrics/particle/momentum/all/assigned/pt_relative_bias": 400.5726623535156, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10205073654651642, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 74781.8671875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010096387937664986, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5118300914764404, "metrics/particle/momentum/all/assigned/pz_bias": 0.003259414341300726, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.577840805053711, "metrics/particle/momentum/all/assigned/samples": 34963.0, "metrics/particle/momentum/all/efficiency": 0.8454222679138184, "metrics/particle/momentum/all/event/energy_bias": -1.344168782234192, "metrics/particle/momentum/all/event/p3_error_mean": 11.766490936279297, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.339295864105225, "metrics/particle/momentum/all/event/px_bias": 0.28817978501319885, "metrics/particle/momentum/all/event/py_bias": 0.1429724544286728, "metrics/particle/momentum/all/event/pz_abs_mean": 8.37051010131836, "metrics/particle/momentum/all/event/pz_bias": 0.3141641914844513, "metrics/particle/momentum/all/event/pz_residual_rms": 15.368570327758789, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007382954470813274, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.044436901807785034, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03637578338384628, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008443786762654781, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08251496404409409, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.000982619938440621, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12779352068901062, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00221342989243567, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8151757717132568, "metrics/particle/momentum/class_1/accepted/samples": 16283.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0641537606716156, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.17787064611911774, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21735194325447083, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1729174256324768, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03459945321083069, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.920125722885132, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0037925199139863253, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23838689923286438, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0020321945194154978, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.456435203552246, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.9498337507247925, "metrics/particle/momentum/class_1/event/energy_bias": 2.0969080924987793, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.519235610961914, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.270443916320801, "metrics/particle/momentum/class_1/event/px_bias": 0.39874517917633057, "metrics/particle/momentum/class_1/event/py_bias": 0.14381393790245056, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.547271728515625, "metrics/particle/momentum/class_1/event/pz_bias": 0.2902478873729706, "metrics/particle/momentum/class_1/event/pz_residual_rms": 9.636119842529297, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.030949674546718597, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1892135590314865, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19049569964408875, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04070981591939926, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23089252412319183, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.039670657366514206, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7113089561462402, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14271128177642822, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.0172119140625, "metrics/particle/momentum/class_2/accepted/samples": 1336.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7640558481216431, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6187089085578918, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5374.75732421875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5374.30126953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3535741865634918, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 274123.96875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07109915465116501, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7334492206573486, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.08207020908594131, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.2468767166137695, "metrics/particle/momentum/class_2/assigned/samples": 2602.0, "metrics/particle/momentum/class_2/efficiency": 0.462283730506897, "metrics/particle/momentum/class_2/event/energy_bias": -7.23317289352417, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.742456436157227, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.559824466705322, "metrics/particle/momentum/class_2/event/px_bias": 0.1265309751033783, "metrics/particle/momentum/class_2/event/py_bias": 0.02598877251148224, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.254721641540527, "metrics/particle/momentum/class_2/event/pz_bias": -1.4664416313171387, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.608793258666992, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016536351293325424, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13249194622039795, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13172107934951782, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03587646037340164, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17355351150035858, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0004939491045661271, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1660294085741043, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007069597952067852, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6500356793403625, "metrics/particle/momentum/class_3/accepted/samples": 12147.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25457215309143066, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.359699010848999, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3330693244934082, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.215701699256897, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13616876304149628, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.623096466064453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006805035751312971, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.436881959438324, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019626250490546227, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4881131649017334, "metrics/particle/momentum/class_3/assigned/samples": 14695.0, "metrics/particle/momentum/class_3/efficiency": 0.7997235059738159, "metrics/particle/momentum/class_3/event/energy_bias": 4.716803073883057, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.202964782714844, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.5454912185668945, "metrics/particle/momentum/class_3/event/px_bias": -0.3143627941608429, "metrics/particle/momentum/class_3/event/py_bias": -0.1829516589641571, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.764782428741455, "metrics/particle/momentum/class_3/event/pz_bias": 1.087093472480774, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.849180221557617, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011143873445689678, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1495496928691864, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10709436237812042, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06113111600279808, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1616419553756714, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.36105507612228394, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04650632292032242, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0491013526916504, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28020936250686646, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6087443828582764, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3547214269638062, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3018276691436768, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14124293625354767, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.506141662597656, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6887131929397583, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1768880933523178, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.079625129699707, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8192090392112732, "metrics/particle/momentum/class_4/event/energy_bias": -1.0354325771331787, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8215374946594238, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3682785034179688, "metrics/particle/momentum/class_4/event/px_bias": -0.02963021770119667, "metrics/particle/momentum/class_4/event/py_bias": 0.16223476827144623, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9684662222862244, "metrics/particle/momentum/class_4/event/pz_bias": 0.28300940990448, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.131818771362305, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004832343198359013, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022515617311000824, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02139851078391075, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.010926821269094944, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.059150632470846176, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12188750505447388, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03493139520287514, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4626770615577698, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02189965546131134, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04164665564894676, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03848041221499443, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0156271830201149, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12139739841222763, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.19180630147457123, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06537139415740967, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8122760653495789, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": 0.11072546988725662, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7984349131584167, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.666145384311676, "metrics/particle/momentum/class_5/event/px_bias": 0.10689664632081985, "metrics/particle/momentum/class_5/event/py_bias": -0.0061133746057748795, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3308925926685333, "metrics/particle/momentum/class_5/event/pz_bias": 0.12025509774684906, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.502238154411316, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.6209619045257568, "step_valid_time": 21413.420397758484, "step_total_time": 21415.041360139847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031595197403493, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06570851692559343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625861503117821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9648026315789474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9636931164777395, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03519736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596324253363965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9618421052631579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9607359947428947, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625861503117821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0034974359781572, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.072603566193716, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2187, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2182, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9600526777875329, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9596314172882844, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9598420013166556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040368582711715666, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9578577699736611, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574374725756911, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9576475751590959, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9600526777875329, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004593028410045, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06886953399259177, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1320, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9416666666666667, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9423805913570887, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.058333333333333334, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9415781487101669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9401515151515152, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9408642911296436, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9430955993930197} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_49000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..1986fc2cce9e65b3adf6f0663e205d47d1ee7ffb --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.320954907161804, "train": {"Classification_binary": 2.2559192180633545, "Classification": 0.017100218683481216, "Regression_pt": 0.10606671124696732, "Regression_eta": 0.00963575392961502, "Regression_sin_phi": 0.006084023509174585, "Regression_cos_phi": 0.006717321462929249, "Regression_energy": 0.11030345410108566, "Total": 6.860847473144531, "Total_unweighted": 2.5118267009966075}, "train_group": {}, "valid": {"Classification_binary": 2.3223562240600586, "Classification": 0.016749978065490723, "Regression_pt": 0.11910318334897359, "Regression_eta": 0.010894390443960825, "Regression_sin_phi": 0.00631976251800855, "Regression_cos_phi": 0.006689983109633128, "Regression_energy": 0.12377464771270752, "Total": 7.289819081624349, "Total_unweighted": 2.605888169258833, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.01653449609875679, "diagnostic/loss/cld_pf/Classification_binary": 2.328084945678711, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.006562856957316399, "diagnostic/loss/cld_pf/Regression_energy": 0.12250909209251404, "diagnostic/loss/cld_pf/Regression_eta": 0.010247955098748207, "diagnostic/loss/cld_pf/Regression_pt": 0.11868106573820114, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005957258865237236, "diagnostic/regression/cld_pf/energy_pred_mean": 0.0863381177186966, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1806187778711319, "diagnostic/regression/cld_pf/energy_residual_mean": 0.059218842536211014, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4058268964290619, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03592769056558609, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.17625734210014343, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06309483200311661, "diagnostic/regression/cld_pf/pt_residual_rms": 0.3987061679363251, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.94994056224823, "metrics/particle/class_1/pid_accuracy": 0.9868256449699402, "metrics/particle/class_2/efficiency": 0.4746496379375458, "metrics/particle/class_2/pid_accuracy": 0.7954872846603394, "metrics/particle/class_3/efficiency": 0.7950729727745056, "metrics/particle/class_3/pid_accuracy": 0.9868708252906799, "metrics/particle/class_4/efficiency": 0.8314833641052246, "metrics/particle/class_4/pid_accuracy": 0.45145630836486816, "metrics/particle/class_5/efficiency": 0.9708737730979919, "metrics/particle/class_5/pid_accuracy": 0.6779999732971191, "metrics/particle/count/bias_mean": -0.45490196347236633, "metrics/particle/count/mae": 2.984313726425171, "metrics/particle/count/prediction_mean": 70.11045837402344, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.04603269323706627, "metrics/particle/event/energy_response_mean": 0.9918274879455566, "metrics/particle/event/met_abs_error": 4.4578962326049805, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036852311342954636, "metrics/particle/event/scalar_pt_response_mean": 1.0012375116348267, "metrics/particle/event/vector_pt_closure": 0.04015910252928734, "metrics/particle/matched/delta_eta_abs_mean": 0.007438801694661379, "metrics/particle/matched/delta_phi_abs_mean": 0.008125151507556438, "metrics/particle/matched/delta_r_mean": 0.012413457967340946, "metrics/particle/matched/energy_relative_abs_mean": 0.08453042805194855, "metrics/particle/matched/pid_accuracy": 0.9715788960456848, "metrics/particle/matched/pt_relative_abs_mean": 0.08001013845205307, "metrics/particle/matching/duplicate_fraction": 0.026978904381394386, "metrics/particle/matching/efficiency": 0.84504234790802, "metrics/particle/matching/f1": 0.8477749824523926, "metrics/particle/matching/purity": 0.8505253195762634, "metrics/particle/matching/target_coverage_dr0p05": 0.8337239027023315, "metrics/particle/matching/target_coverage_dr0p10": 0.87660813331604, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.746436357498169, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.877858579158783, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014109265990555286, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07967951148748398, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07694356143474579, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015524900518357754, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12802234292030334, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004637888167053461, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3341119885444641, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.015492287464439869, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4470100402832031, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8409.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2231619507074356, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7865822911262512, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7639719843864441, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6637235283851624, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10618839412927628, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.120729923248291, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011685804463922977, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6314846277236938, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0427047535777092, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4396276473999023, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9841.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8295353651046753, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.49861425161361694, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.878856658935547, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.591085910797119, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.020074594765901566, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.18903043866157532, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.07476806640625, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.1671156883239746, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.948370933532715, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022820837795734406, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09191250056028366, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09083953499794006, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010312783531844616, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14206276834011078, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02267303131520748, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9772799015045166, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19386817514896393, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1980903148651123, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1676.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7182933688163757, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4671896696090698, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5913.85400390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5913.70068359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24089306592941284, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 298645.6875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07794751226902008, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5541536808013916, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1541978120803833, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.634588718414307, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2553.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6096762418746948, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.8155086040496826, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.909573078155518, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8012042045593262, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.055232007056474686, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0566340908408165, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.792304039001465, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.16539400815963745, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.861917495727539, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010707193054258823, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08704756200313568, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08143214881420135, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01680011674761772, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13535119593143463, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00029859659844078124, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10403116792440414, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0042580957524478436, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4509032368659973, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20094.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12654756009578705, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5753905773162842, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.44667500257492065, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.34351593255996704, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08335186541080475, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.018054008483887, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0020015123300254345, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22003188729286194, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.000983005273155868, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4986859560012817, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22483.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8788872957229614, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.33090364933013916, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.521154403686523, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.723214149475098, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.1576271504163742, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.26138412952423096, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.3604636192321777, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3438425362110138, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.006798267364502, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012327873148024082, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08526472747325897, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08070390671491623, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016084518283605576, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1337367296218872, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002750256797298789, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21663643419742584, "metrics/particle/momentum/all/accepted/pz_bias": -0.01791841723024845, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1343705654144287, "metrics/particle/momentum/all/accepted/samples": 30179.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1971244364976883, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7002608180046082, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 433.3982238769531, "metrics/particle/momentum/all/assigned/pt_relative_bias": 433.292236328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10132752358913422, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 80800.140625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010293316096067429, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5069866180419922, "metrics/particle/momentum/all/assigned/pz_bias": -0.02397068217396736, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5197958946228027, "metrics/particle/momentum/all/assigned/samples": 34877.0, "metrics/particle/momentum/all/efficiency": 0.8441914319992065, "metrics/particle/momentum/all/event/energy_bias": -1.9832191467285156, "metrics/particle/momentum/all/event/p3_error_mean": 12.285244941711426, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.395114421844482, "metrics/particle/momentum/all/event/px_bias": -0.19278456270694733, "metrics/particle/momentum/all/event/py_bias": -0.3937804400920868, "metrics/particle/momentum/all/event/pz_abs_mean": 8.942893981933594, "metrics/particle/momentum/all/event/pz_bias": -1.6763521432876587, "metrics/particle/momentum/all/event/pz_residual_rms": 16.286760330200195, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007682998199015856, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0424138568341732, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03538923338055611, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006146086845546961, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08132852613925934, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001229407382197678, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12721507251262665, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01282359380275011, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8238092660903931, "metrics/particle/momentum/class_1/accepted/samples": 16268.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06481415778398514, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2151893824338913, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.24869510531425476, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20254139602184296, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.035535067319869995, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.443212032318115, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0037343914154917, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2338484823703766, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.024587338790297508, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.4303598403930664, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9489587545394897, "metrics/particle/momentum/class_1/event/energy_bias": 1.9245240688323975, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.83439826965332, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.41944694519043, "metrics/particle/momentum/class_1/event/px_bias": 0.20703649520874023, "metrics/particle/momentum/class_1/event/py_bias": 0.10631623864173889, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.752320766448975, "metrics/particle/momentum/class_1/event/pz_bias": -0.4474814534187317, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.237279891967773, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03134604170918465, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18845491111278534, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18943749368190765, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04892055317759514, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2290578931570053, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0391722097992897, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7275809049606323, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.21428212523460388, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.046888828277588, "metrics/particle/momentum/class_2/accepted/samples": 1353.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7973951101303101, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6561061143875122, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5806.904296875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5806.4541015625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3482108414173126, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 295991.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07156598567962646, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7539775371551514, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18699368834495544, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.124500274658203, "metrics/particle/momentum/class_2/assigned/samples": 2599.0, "metrics/particle/momentum/class_2/efficiency": 0.4681660830974579, "metrics/particle/momentum/class_2/event/energy_bias": -7.293651580810547, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.76593017578125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.276370525360107, "metrics/particle/momentum/class_2/event/px_bias": -0.20060016214847565, "metrics/particle/momentum/class_2/event/py_bias": -0.1317131221294403, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.505812644958496, "metrics/particle/momentum/class_2/event/pz_bias": -1.988287329673767, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.110469818115234, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016558025032281876, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1307307928800583, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.12968707084655762, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.035929858684539795, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17084944248199463, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007438016473315656, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.16427014768123627, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.003545304061844945, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6468616127967834, "metrics/particle/momentum/class_3/accepted/samples": 12100.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24554820358753204, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.248815655708313, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2841185331344604, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1669498682022095, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13419558107852936, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.091535568237305, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007185382768511772, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4303693473339081, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.003983017988502979, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.449230432510376, "metrics/particle/momentum/class_3/assigned/samples": 14613.0, "metrics/particle/momentum/class_3/efficiency": 0.7966291308403015, "metrics/particle/momentum/class_3/event/energy_bias": 4.4118123054504395, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.408863067626953, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.681458473205566, "metrics/particle/momentum/class_3/event/px_bias": -0.26363152265548706, "metrics/particle/momentum/class_3/event/py_bias": -0.4793778657913208, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.92641019821167, "metrics/particle/momentum/class_3/event/pz_bias": 0.3819034695625305, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.211573600769043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011744312010705471, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14699643850326538, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1059052050113678, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05352282524108887, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15970122814178467, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003448275849223137, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4212178587913513, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0052461749874055386, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3615344762802124, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2892579436302185, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.1853432655334473, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.278018593788147, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.220190167427063, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15536722540855408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.577537536621094, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5418989658355713, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07725223153829575, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7432864904403687, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8192090392112732, "metrics/particle/momentum/class_4/event/energy_bias": -1.1152167320251465, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7714588642120361, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3279049396514893, "metrics/particle/momentum/class_4/event/px_bias": -0.05907764285802841, "metrics/particle/momentum/class_4/event/py_bias": 0.11756531149148941, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9433506727218628, "metrics/particle/momentum/class_4/event/pz_bias": 0.27014774084091187, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.121992111206055, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005278155207633972, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02240712381899357, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021524988114833832, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008016240783035755, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0591038316488266, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1255819946527481, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0050307572819292545, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.45425525307655334, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007515480276197195, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04223557189106941, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03917234390974045, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01835746504366398, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12492142617702484, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.20849375426769257, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08209312707185745, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9659604430198669, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.08931256085634232, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8559963703155518, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6980729699134827, "metrics/particle/momentum/class_5/event/px_bias": 0.12348826974630356, "metrics/particle/momentum/class_5/event/py_bias": -0.006571043282747269, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3689174950122833, "metrics/particle/momentum/class_5/event/pz_bias": 0.10736542195081711, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.6107696294784546, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.82151198387146, "step_valid_time": 21860.70401263237, "step_total_time": 21862.525525331497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000838521506941, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06755316236583031, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9632425336396455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9673698088332234, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9653017595790166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03263019116677653, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9606170003281916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9647330257086355, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9626706133859563, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9632425336396455, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002492964859793, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06982406071850189, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2187, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2180, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9600526777875329, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9604743083003953, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9602634467618002, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039525691699604744, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9569798068481123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574000878348704, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9571899012074643, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9600526777875329, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058780257743418, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07207163522611226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1239, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9415781487101669, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9444444444444444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9430091185410334, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05555555555555555, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9400606980273141, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9429223744292238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9414893617021277, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9415781487101669} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_5000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..3fb989f9070d4a525cf4a5364ce49e4facd7c8c0 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23683213338385753, "train": {"Classification_binary": 3.186685562133789, "Classification": 0.025166105479002, "Regression_pt": 0.16689300537109375, "Regression_eta": 0.009807479567825794, "Regression_sin_phi": 0.006987044122070074, "Regression_cos_phi": 0.006443746853619814, "Regression_energy": 0.16869090497493744, "Total": 9.581254005432129, "Total_unweighted": 3.570673848502338}, "train_group": {}, "valid": {"Classification_binary": 3.3358004887898765, "Classification": 0.02551328142484029, "Regression_pt": 0.18221014738082886, "Regression_eta": 0.014105459054311117, "Regression_sin_phi": 0.007862089201807976, "Regression_cos_phi": 0.008329298968116442, "Regression_energy": 0.18789943059285483, "Total": 10.539953231811523, "Total_unweighted": 3.761720195412636, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.025665977969765663, "diagnostic/loss/cld_pf/Classification_binary": 3.317366361618042, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007821009494364262, "diagnostic/loss/cld_pf/Regression_energy": 0.18711957335472107, "diagnostic/loss/cld_pf/Regression_eta": 0.013030115514993668, "diagnostic/loss/cld_pf/Regression_pt": 0.18333721160888672, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.00728426780551672, "diagnostic/regression/cld_pf/energy_pred_mean": 0.10833080857992172, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.23002441227436066, "diagnostic/regression/cld_pf/energy_residual_mean": 0.08121152967214584, "diagnostic/regression/cld_pf/energy_residual_rms": 0.48055073618888855, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.05593631789088249, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.22111371159553528, "diagnostic/regression/cld_pf/pt_residual_mean": 0.08310345560312271, "diagnostic/regression/cld_pf/pt_residual_rms": 0.47722211480140686, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.91001957654953, "metrics/particle/class_1/pid_accuracy": 0.981467068195343, "metrics/particle/class_2/efficiency": 0.3287000060081482, "metrics/particle/class_2/pid_accuracy": 0.5337954759597778, "metrics/particle/class_3/efficiency": 0.7276283502578735, "metrics/particle/class_3/pid_accuracy": 0.979432225227356, "metrics/particle/class_4/efficiency": 0.8395559787750244, "metrics/particle/class_4/pid_accuracy": 0.21033653616905212, "metrics/particle/class_5/efficiency": 0.9514563083648682, "metrics/particle/class_5/pid_accuracy": 0.45306122303009033, "metrics/particle/count/bias_mean": -2.205228805541992, "metrics/particle/count/mae": 4.449673175811768, "metrics/particle/count/prediction_mean": 68.3601303100586, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.07358723878860474, "metrics/particle/event/energy_response_mean": 0.960598886013031, "metrics/particle/event/met_abs_error": 6.880429744720459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0614265538752079, "metrics/particle/event/scalar_pt_response_mean": 0.9739903807640076, "metrics/particle/event/vector_pt_closure": 0.06185479089617729, "metrics/particle/matched/delta_eta_abs_mean": 0.014611239545047283, "metrics/particle/matched/delta_phi_abs_mean": 0.012506010942161083, "metrics/particle/matched/delta_r_mean": 0.021549947559833527, "metrics/particle/matched/energy_relative_abs_mean": 0.10215286910533905, "metrics/particle/matched/pid_accuracy": 0.9548231363296509, "metrics/particle/matched/pt_relative_abs_mean": 0.08951758593320847, "metrics/particle/matching/duplicate_fraction": 0.0392003133893013, "metrics/particle/matching/efficiency": 0.7854397296905518, "metrics/particle/matching/f1": 0.7979073524475098, "metrics/particle/matching/purity": 0.8107772469520569, "metrics/particle/matching/target_coverage_dr0p05": 0.7293104529380798, "metrics/particle/matching/target_coverage_dr0p10": 0.8237113952636719, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.679192304611206, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8251470327377319, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02284841239452362, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09678613394498825, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08676952123641968, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01792188175022602, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13885772228240967, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005547671113163233, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4085574150085449, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0057509043253958225, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6719584465026855, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7751.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3136589229106903, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1620336771011353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2624753713607788, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1436059474945068, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14220327138900757, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.147598266601562, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01851659268140793, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9015111327171326, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.11634324491024017, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2503116130828857, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9613.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7646246552467346, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.5998361110687256, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.877510070800781, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8824830055236816, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5300983190536499, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.4933887720108032, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.66724967956543, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.668847382068634, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.342646598815918, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.031568169593811035, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.11001363396644592, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10362696647644043, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03212554380297661, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15589195489883423, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0195258017629385, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0236462354660034, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.017675058916211128, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.315136432647705, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1434.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8386135697364807, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6675113439559937, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 15192.119140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 15191.9619140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2968168556690216, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 747027.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08763951808214188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.467893362045288, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.19203829765319824, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.24368953704834, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2419.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5216442346572876, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.008492469787598, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.223073959350586, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3642122745513916, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.3115800619125366, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.12700991332530975, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.037906646728516, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.780388593673706, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.172321319580078, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020407481119036674, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.10488369315862656, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09093444794416428, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014378787949681282, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1462104469537735, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006323111010715365, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13389480113983154, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 3.768285023397766e-05, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5844658017158508, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18978.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18634618818759918, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8147120475769043, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5969019532203674, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4700008034706116, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11397373676300049, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.032378673553467, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0044217840768396854, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33185040950775146, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.029587315395474434, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4938256740570068, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22163.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8300747871398926, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -4.4277567863464355, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.656021118164062, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.05737018585205, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9986205697059631, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6748929619789124, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.490660190582275, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0248053073883057, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.313690185546875, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02164755016565323, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.10291629284620285, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09043445438146591, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016257543116807938, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14473959803581238, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029471293091773987, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2547914683818817, "metrics/particle/momentum/all/accepted/pz_bias": -0.0006573900463990867, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2486577033996582, "metrics/particle/momentum/all/accepted/samples": 28163.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26827898621559143, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9726803302764893, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1075.45263671875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1075.3258056640625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13484427332878113, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 198688.828125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014271092601120472, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7138429284095764, "metrics/particle/momentum/all/assigned/pz_bias": 0.06546840071678162, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6550533771514893, "metrics/particle/momentum/all/assigned/samples": 34195.0, "metrics/particle/momentum/all/efficiency": 0.787798285484314, "metrics/particle/momentum/all/event/energy_bias": -10.03608512878418, "metrics/particle/momentum/all/event/p3_error_mean": 19.728771209716797, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.362195014953613, "metrics/particle/momentum/all/event/px_bias": -1.8402990102767944, "metrics/particle/momentum/all/event/py_bias": 1.0412718057632446, "metrics/particle/momentum/all/event/pz_abs_mean": 14.259395599365234, "metrics/particle/momentum/all/event/pz_bias": 4.474040985107422, "metrics/particle/momentum/all/event/pz_residual_rms": 25.104297637939453, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012390704825520515, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.07090046256780624, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.049098577350378036, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01699058711528778, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.10176154226064682, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014758727047592402, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16784588992595673, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.010500858537852764, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8418546319007874, "metrics/particle/momentum/class_1/accepted/samples": 15584.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11756264418363571, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32809382677078247, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3777894377708435, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3096841275691986, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0690520629286766, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.474956512451172, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00782162044197321, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.437906414270401, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04274298995733261, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.0538201332092285, "metrics/particle/momentum/class_1/assigned/samples": 17132.0, "metrics/particle/momentum/class_1/efficiency": 0.9090591073036194, "metrics/particle/momentum/class_1/event/energy_bias": 2.7443861961364746, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.222984313964844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.691821098327637, "metrics/particle/momentum/class_1/event/px_bias": -0.6405571699142456, "metrics/particle/momentum/class_1/event/py_bias": 0.7358454465866089, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.509184837341309, "metrics/particle/momentum/class_1/event/pz_bias": 1.5220046043395996, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.12961769104004, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04455779865384102, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.205201655626297, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20437180995941162, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06244194135069847, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24316267669200897, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0518292672932148, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.065006971359253, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.19501107931137085, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.702938079833984, "metrics/particle/momentum/class_2/accepted/samples": 984.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0188952684402466, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8881312012672424, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 15803.9169921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 15803.41015625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4116128981113434, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 761979.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09376344084739685, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.664257764816284, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2035694122314453, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.034174919128418, "metrics/particle/momentum/class_2/assigned/samples": 2325.0, "metrics/particle/momentum/class_2/efficiency": 0.3404844403266907, "metrics/particle/momentum/class_2/event/energy_bias": -16.058393478393555, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.552574157714844, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.306102752685547, "metrics/particle/momentum/class_2/event/px_bias": -0.7022685408592224, "metrics/particle/momentum/class_2/event/py_bias": -0.2587786018848419, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.755279541015625, "metrics/particle/momentum/class_2/event/pz_bias": 1.8238983154296875, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.42452621459961, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.032918162643909454, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1375381201505661, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13852903246879578, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021659553050994873, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1802499145269394, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000538744730874896, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20761357247829437, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00476588262245059, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9287129044532776, "metrics/particle/momentum/class_3/accepted/samples": 11137.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33135896921157837, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7396275997161865, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6908875703811646, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5531229972839355, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17004361748695374, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.405282974243164, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009216265752911568, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5668003559112549, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06534374505281448, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.1853811740875244, "metrics/particle/momentum/class_3/assigned/samples": 14214.0, "metrics/particle/momentum/class_3/efficiency": 0.7332279682159424, "metrics/particle/momentum/class_3/event/energy_bias": 6.1673264503479, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.722768783569336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.631543159484863, "metrics/particle/momentum/class_3/event/px_bias": -0.6426041722297668, "metrics/particle/momentum/class_3/event/py_bias": 0.4919688105583191, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.244847297668457, "metrics/particle/momentum/class_3/event/pz_bias": 0.5295557975769043, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.423447608947754, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015340965241193771, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1806364506483078, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11017667502164841, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.038372576236724854, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.15657418966293335, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010273972526192665, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6101445555686951, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.191886767745018, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1844322681427, "metrics/particle/momentum/class_4/accepted/samples": 292.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2149241715669632, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.3825957775115967, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3518518209457397, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2767385244369507, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.13960114121437073, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.476877689361572, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014245014637708664, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.814824104309082, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2695540189743042, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.74192476272583, "metrics/particle/momentum/class_4/assigned/samples": 351.0, "metrics/particle/momentum/class_4/efficiency": 0.8248587846755981, "metrics/particle/momentum/class_4/event/energy_bias": -2.385697841644287, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.051046371459961, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1692605018615723, "metrics/particle/momentum/class_4/event/px_bias": 0.07823708653450012, "metrics/particle/momentum/class_4/event/py_bias": 0.14979635179042816, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7498750686645508, "metrics/particle/momentum/class_4/event/pz_bias": 0.3615437150001526, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.45875358581543, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009814792312681675, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.04272718355059624, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03422853350639343, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012623654678463936, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.09318412095308304, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.22687149047851562, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0408322811126709, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7865973114967346, "metrics/particle/momentum/class_5/accepted/samples": 166.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03127427399158478, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06726130843162537, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.059255246073007584, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.018224310129880905, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17573882639408112, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.2644210457801819, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.05613114684820175, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.8252731561660767, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9595375657081604, "metrics/particle/momentum/class_5/event/energy_bias": -0.5037055015563965, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.215977668762207, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9247967004776001, "metrics/particle/momentum/class_5/event/px_bias": 0.06689378619194031, "metrics/particle/momentum/class_5/event/py_bias": -0.07756024599075317, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6338573694229126, "metrics/particle/momentum/class_5/event/pz_bias": 0.23703879117965698, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.6221516132354736, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4446542263031006, "step_valid_time": 2225.258168697357, "step_total_time": 2226.7028234004974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9928360269482386, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11528870142331482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2856, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2840, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9373153921890384, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9416419386745796, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9394736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.058358061325420374, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9320643255661306, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9363666336960106, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9342105263157895, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9373153921890384, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.990755997242649, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12275433128572077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2101, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2086, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9223002633889377, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9321206743566992, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9271844660194175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06787932564330079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9157155399473222, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9254658385093167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9205648720211828, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9223002633889377, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9913310346374591, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1305268037633407, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1203, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9127465857359636, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9155251141552512, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9141337386018237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08447488584474885, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9081942336874052, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.910958904109589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9095744680851063, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9127465857359636} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_50000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..68884882838a8e98576bfb035d28bfee5ac1506c --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3683213338385753, "train": {"Classification_binary": 2.270394802093506, "Classification": 0.016064846888184547, "Regression_pt": 0.11500262469053268, "Regression_eta": 0.012551912106573582, "Regression_sin_phi": 0.007289600092917681, "Regression_cos_phi": 0.00657888688147068, "Regression_energy": 0.11971940100193024, "Total": 7.247401237487793, "Total_unweighted": 2.5476020737551153}, "train_group": {}, "valid": {"Classification_binary": 2.3185818990071616, "Classification": 0.01685275261600812, "Regression_pt": 0.11846170822779338, "Regression_eta": 0.010778031001488367, "Regression_sin_phi": 0.0063181072473526, "Regression_cos_phi": 0.006697735438744227, "Regression_energy": 0.12330185373624165, "Total": 7.272661209106445, "Total_unweighted": 2.60099208727479, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.016674179583787918, "diagnostic/loss/cld_pf/Classification_binary": 2.3251919746398926, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.0065064518712460995, "diagnostic/loss/cld_pf/Regression_energy": 0.12180779874324799, "diagnostic/loss/cld_pf/Regression_eta": 0.009932005777955055, "diagnostic/loss/cld_pf/Regression_pt": 0.11789289116859436, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.005933527834713459, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08333810418844223, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.1805351972579956, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05621882528066635, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4022805690765381, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.031655021011829376, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.1758209764957428, "diagnostic/regression/cld_pf/pt_residual_mean": 0.058822162449359894, "diagnostic/regression/cld_pf/pt_residual_rms": 0.39451363682746887, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9509946703910828, "metrics/particle/class_1/pid_accuracy": 0.9876261949539185, "metrics/particle/class_2/efficiency": 0.4789791405200958, "metrics/particle/class_2/pid_accuracy": 0.7894862294197083, "metrics/particle/class_3/efficiency": 0.7964574694633484, "metrics/particle/class_3/pid_accuracy": 0.9871143698692322, "metrics/particle/class_4/efficiency": 0.8294652104377747, "metrics/particle/class_4/pid_accuracy": 0.42214110493659973, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.6918489336967468, "metrics/particle/count/bias_mean": -0.07581699639558792, "metrics/particle/count/mae": 2.966012954711914, "metrics/particle/count/prediction_mean": 70.48954010009766, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.045057132840156555, "metrics/particle/event/energy_response_mean": 0.9920868873596191, "metrics/particle/event/met_abs_error": 4.353595733642578, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036033980548381805, "metrics/particle/event/scalar_pt_response_mean": 0.9989187717437744, "metrics/particle/event/vector_pt_closure": 0.03938758745789528, "metrics/particle/matched/delta_eta_abs_mean": 0.007141394540667534, "metrics/particle/matched/delta_phi_abs_mean": 0.008170626126229763, "metrics/particle/matched/delta_r_mean": 0.012215782888233662, "metrics/particle/matched/energy_relative_abs_mean": 0.08556187897920609, "metrics/particle/matched/pid_accuracy": 0.9715948700904846, "metrics/particle/matched/pt_relative_abs_mean": 0.0806594118475914, "metrics/particle/matching/duplicate_fraction": 0.027325240895152092, "metrics/particle/matching/efficiency": 0.8464965224266052, "metrics/particle/matching/f1": 0.8469515442848206, "metrics/particle/matching/purity": 0.8474070429801941, "metrics/particle/matching/target_coverage_dr0p05": 0.8348075747489929, "metrics/particle/matching/target_coverage_dr0p10": 0.8779512047767639, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7461955547332764, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8797295689582825, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.013989916071295738, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08223265409469604, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07910003513097763, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016671352088451385, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1309857815504074, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0046345810405910015, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3413274884223938, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0046003833413124084, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4349666833877563, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8415.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2295062392950058, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8600408434867859, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8600226044654846, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7575463652610779, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1090393215417862, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.756543159484863, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012464531697332859, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6373964548110962, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.002503420226275921, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.369727849960327, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9868.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8301272392272949, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.21708834171295166, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.706849575042725, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5368831157684326, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.007227197289466858, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.0703723132610321, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.91928243637085, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.07812385261058807, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.981422424316406, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02150142937898636, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09149351716041565, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09028125554323196, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016710713505744934, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14136919379234314, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02469879575073719, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9526744484901428, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17667555809020996, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0523903369903564, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1660.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.718024492263794, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3773837089538574, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5844.12548828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5843.96923828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24190402030944824, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 295742.96875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07179086655378342, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6239490509033203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.00898828823119402, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.916467189788818, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2563.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6038559675216675, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.400052785873413, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 4.879947185516357, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.7788305878639221, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07015734910964966, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1015256941318512, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 4.765523433685303, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.643019437789917, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.629672050476074, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01050911657512188, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08796714246273041, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0820838212966919, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013751554302871227, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13661769032478333, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00029748622910119593, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10318335890769958, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0004904580418951809, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.4717339277267456, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20169.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1290384829044342, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5653260946273804, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.43560487031936646, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.32854336500167847, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08156563341617584, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.652453899383545, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0022188692819327116, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21823014318943024, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.010235356166958809, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1374785900115967, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22534.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8821676969528198, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.7362520694732666, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.482473373413086, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.657711029052734, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.17853331565856934, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.004305630922317505, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.412872791290283, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.18055005371570587, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.061609268188477, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012080936692655087, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08656514436006546, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0817035511136055, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014726370573043823, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13534235954284668, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028435392305254936, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21606981754302979, "metrics/particle/momentum/all/accepted/pz_bias": -0.01065010018646717, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1102765798568726, "metrics/particle/momentum/all/accepted/samples": 30244.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20056678354740143, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7080273628234863, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 428.9087829589844, "metrics/particle/momentum/all/assigned/pt_relative_bias": 428.7994079589844, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10107249766588211, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 80070.3984375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010210210457444191, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.512873113155365, "metrics/particle/momentum/all/assigned/pz_bias": 0.005231024697422981, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6481282711029053, "metrics/particle/momentum/all/assigned/samples": 34965.0, "metrics/particle/momentum/all/efficiency": 0.8460096716880798, "metrics/particle/momentum/all/event/energy_bias": -2.353393077850342, "metrics/particle/momentum/all/event/p3_error_mean": 11.977392196655273, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.24188232421875, "metrics/particle/momentum/all/event/px_bias": 0.10114876925945282, "metrics/particle/momentum/all/event/py_bias": 0.035459004342556, "metrics/particle/momentum/all/event/pz_abs_mean": 8.75848388671875, "metrics/particle/momentum/all/event/pz_bias": 0.9016933441162109, "metrics/particle/momentum/all/event/pz_residual_rms": 15.64345645904541, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007328768726438284, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04324311390519142, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03542450815439224, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00547384936362505, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08126607537269592, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001166646252386272, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12778601050376892, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.008981261402368546, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8071711659431458, "metrics/particle/momentum/class_1/accepted/samples": 16286.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06373406201601028, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1944504678249359, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21592409908771515, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1685035228729248, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03395764157176018, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.966082811355591, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003442441113293171, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2340095341205597, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.004728150554001331, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.585861086845398, "metrics/particle/momentum/class_1/assigned/samples": 17139.0, "metrics/particle/momentum/class_1/efficiency": 0.950008749961853, "metrics/particle/momentum/class_1/event/energy_bias": 1.5520503520965576, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.04248046875, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.569674015045166, "metrics/particle/momentum/class_1/event/px_bias": 0.3979467749595642, "metrics/particle/momentum/class_1/event/py_bias": 0.1781364530324936, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.823461532592773, "metrics/particle/momentum/class_1/event/pz_bias": -0.18767262995243073, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.843528747558594, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031359340995550156, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1893809586763382, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1908905953168869, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04455401748418808, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23128579556941986, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04319180175662041, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6979632377624512, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.18105252087116241, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9521634578704834, "metrics/particle/momentum/class_2/accepted/samples": 1366.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8185228705406189, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6935974955558777, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 5733.130859375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 5732.66943359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34992343187332153, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 292955.84375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07695252448320389, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8756210803985596, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07296790927648544, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.785918235778809, "metrics/particle/momentum/class_2/assigned/samples": 2612.0, "metrics/particle/momentum/class_2/efficiency": 0.47266435623168945, "metrics/particle/momentum/class_2/event/energy_bias": -7.559256553649902, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.566360473632812, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.290033340454102, "metrics/particle/momentum/class_2/event/px_bias": 0.06531281024217606, "metrics/particle/momentum/class_2/event/py_bias": 0.02426941692829132, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.268204689025879, "metrics/particle/momentum/class_2/event/pz_bias": -0.8543145656585693, "metrics/particle/momentum/class_2/event/pz_residual_rms": 13.6981782913208, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016402004286646843, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13244087994098663, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13170623779296875, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.032700974494218826, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17343769967556, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000659358745906502, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1662573367357254, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0051951939240098, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.64497971534729, "metrics/particle/momentum/class_3/accepted/samples": 12133.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2507624626159668, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.245848298072815, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1698379516601562, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.048300862312317, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13494928181171417, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.33935546875, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006400217767804861, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4216793477535248, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03143387660384178, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3431739807128906, "metrics/particle/momentum/class_3/assigned/samples": 14687.0, "metrics/particle/momentum/class_3/efficiency": 0.7988017797470093, "metrics/particle/momentum/class_3/event/energy_bias": 4.787432670593262, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.3873291015625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.641161918640137, "metrics/particle/momentum/class_3/event/px_bias": -0.2737575173377991, "metrics/particle/momentum/class_3/event/py_bias": -0.28217339515686035, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.949290752410889, "metrics/particle/momentum/class_3/event/pz_bias": 1.3603328466415405, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.609227180480957, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011530157178640366, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1530245691537857, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10964401811361313, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06432782113552094, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16389374434947968, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.33929431438446045, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.010474753566086292, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9400923252105713, "metrics/particle/momentum/class_4/accepted/samples": 291.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.277525395154953, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.6908507347106934, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.738919734954834, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.6888742446899414, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14406779408454895, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 27.318359375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5191887021064758, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.003933502361178398, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7032896280288696, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8220338821411133, "metrics/particle/momentum/class_4/event/energy_bias": -1.227649450302124, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7479753494262695, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.333968162536621, "metrics/particle/momentum/class_4/event/px_bias": -0.1847706139087677, "metrics/particle/momentum/class_4/event/py_bias": 0.1316443681716919, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9095324277877808, "metrics/particle/momentum/class_4/event/pz_bias": 0.46621108055114746, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.3412258625030518, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004892350640147924, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021958595141768456, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020618965849280357, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009628962725400925, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.058959197252988815, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.10916668176651001, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.032163750380277634, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.42681461572647095, "metrics/particle/momentum/class_5/accepted/samples": 168.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007544562220573425, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04322725534439087, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03937884047627449, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0214565247297287, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13051000237464905, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.1953674554824829, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.049312617629766464, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.920392632484436, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9710982441902161, "metrics/particle/momentum/class_5/event/energy_bias": 0.09403002262115479, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7654101848602295, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6413869261741638, "metrics/particle/momentum/class_5/event/px_bias": 0.09641730785369873, "metrics/particle/momentum/class_5/event/py_bias": -0.016417816281318665, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.3086916208267212, "metrics/particle/momentum/class_5/event/pz_bias": 0.11713670194149017, "metrics/particle/momentum/class_5/event/pz_residual_rms": 1.4265944957733154, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.807325839996338, "step_valid_time": 22309.517141819, "step_total_time": 22311.324468135834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0012307982271196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0686364870520686, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2944, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9661962586150312, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.97001647446458, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9681025978296613, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.029983525535420098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9629143419757138, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9667215815485997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9648142058533378, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9661962586150312, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002932205276438, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06983708273688716, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2181, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9574187884108868, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959524857017158, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.958470665787739, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040475142982842056, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9543459174714662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9564452265728113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.955394418809053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9574187884108868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038888833514774, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06916537625220642, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9430955993930197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05690440060698027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9423368740515933, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9423368740515933, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9423368740515933, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9430955993930197} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_6000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..ca8fce6750f300fa1f4bc4002dd5bcdaaa660e69 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28419856006062905, "train": {"Classification_binary": 3.2401652336120605, "Classification": 0.02558322250843048, "Regression_pt": 0.16750667989253998, "Regression_eta": 0.010627173818647861, "Regression_sin_phi": 0.008155525662004948, "Regression_cos_phi": 0.008275967091321945, "Regression_energy": 0.1719948947429657, "Total": 9.940759658813477, "Total_unweighted": 3.6323086973279715}, "train_group": {}, "valid": {"Classification_binary": 3.1826775868733725, "Classification": 0.02352648725112279, "Regression_pt": 0.17618036270141602, "Regression_eta": 0.013610387841860453, "Regression_sin_phi": 0.00778371033569177, "Regression_cos_phi": 0.008245906482140223, "Regression_energy": 0.1822135845820109, "Total": 10.124881108601889, "Total_unweighted": 3.5942380260676146, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.023633627220988274, "diagnostic/loss/cld_pf/Classification_binary": 3.1785202026367188, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007715489715337753, "diagnostic/loss/cld_pf/Regression_energy": 0.18425782024860382, "diagnostic/loss/cld_pf/Regression_eta": 0.012628857046365738, "diagnostic/loss/cld_pf/Regression_pt": 0.18005546927452087, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007175789680331945, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09265691041946411, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21727877855300903, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06553763151168823, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4699420630931854, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.048765163868665695, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.21371227502822876, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07593230158090591, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4654996991157532, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9256583452224731, "metrics/particle/class_1/pid_accuracy": 0.9819250702857971, "metrics/particle/class_2/efficiency": 0.3845277428627014, "metrics/particle/class_2/pid_accuracy": 0.6471111178398132, "metrics/particle/class_3/efficiency": 0.7461102604866028, "metrics/particle/class_3/pid_accuracy": 0.9796176552772522, "metrics/particle/class_4/efficiency": 0.8567103743553162, "metrics/particle/class_4/pid_accuracy": 0.19434629380702972, "metrics/particle/class_5/efficiency": 0.9572815299034119, "metrics/particle/class_5/pid_accuracy": 0.4787018299102783, "metrics/particle/count/bias_mean": 0.6294117569923401, "metrics/particle/count/mae": 3.872549057006836, "metrics/particle/count/prediction_mean": 71.19477081298828, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06457462906837463, "metrics/particle/event/energy_response_mean": 1.0033314228057861, "metrics/particle/event/met_abs_error": 6.849846363067627, "metrics/particle/event/scalar_pt_relative_abs_error": 0.055508099496364594, "metrics/particle/event/scalar_pt_response_mean": 1.017458200454712, "metrics/particle/event/vector_pt_closure": 0.05888678506016731, "metrics/particle/matched/delta_eta_abs_mean": 0.0153864286839962, "metrics/particle/matched/delta_phi_abs_mean": 0.013093238696455956, "metrics/particle/matched/delta_r_mean": 0.02256138250231743, "metrics/particle/matched/energy_relative_abs_mean": 0.0945204496383667, "metrics/particle/matched/pid_accuracy": 0.9574896097183228, "metrics/particle/matched/pt_relative_abs_mean": 0.08974077552556992, "metrics/particle/matching/duplicate_fraction": 0.04334055632352829, "metrics/particle/matching/efficiency": 0.8055110573768616, "metrics/particle/matching/f1": 0.8019345998764038, "metrics/particle/matching/purity": 0.798389732837677, "metrics/particle/matching/target_coverage_dr0p05": 0.75323486328125, "metrics/particle/matching/target_coverage_dr0p10": 0.8431806564331055, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6992266178131104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8477839827537537, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024780316278338432, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08967850357294083, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08777672052383423, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02278381586074829, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14096206426620483, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005522781517356634, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4077031910419464, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05817073583602905, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6423536539077759, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7967.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3268064558506012, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3189045190811157, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.368485689163208, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.248924970626831, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14317379891872406, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.956400871276855, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01823677495121956, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8859396576881409, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05501697212457657, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2684409618377686, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9925.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.785932719707489, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.503519058227539, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.099451065063477, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.533495903015137, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15644025802612305, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.13250842690467834, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.693977355957031, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7909607887268066, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.43601417541504, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.033182110637426376, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09206660091876984, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0949394553899765, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02988547459244728, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1465928554534912, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021145975217223167, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9027379155158997, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.033670585602521896, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9573559761047363, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1466.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8601770401000977, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.948182225227356, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 15367.2939453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 15367.1240234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3012668192386627, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 772126.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0926365777850151, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.275874614715576, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.21304425597190857, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.740744590759277, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2526.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5332848429679871, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.2325029373168945, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.039203643798828, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4268338680267334, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1478850245475769, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06773573160171509, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.836259841918945, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.1621251255273819, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.50133514404297, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021009553223848343, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09900359809398651, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.09255915880203247, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017353737726807594, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14673052728176117, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007203128188848495, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13851533830165863, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03449169173836708, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5695788264274597, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19436.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1855601817369461, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8238312602043152, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6390755772590637, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5147058367729187, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1100376695394516, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.881099224090576, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004165743477642536, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3317160904407501, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0314568467438221, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.62641978263855, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22565.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8501071333885193, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.572680950164795, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.215059280395508, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.505455493927002, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.1263419389724731, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7181221842765808, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.414004802703857, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6009371280670166, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.507209777832031, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02266830764710903, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09607787430286407, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09136022627353668, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019488655030727386, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1451544314622879, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003082891693338752, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2516114115715027, "metrics/particle/momentum/all/accepted/pz_bias": 0.04098472371697426, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1861398220062256, "metrics/particle/momentum/all/accepted/samples": 28869.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.27426108717918396, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.045264720916748, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1109.37255859375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1109.2464599609375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13322481513023376, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 207382.453125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01453621219843626, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7011932730674744, "metrics/particle/momentum/all/assigned/pz_bias": 0.02049684152007103, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.783891201019287, "metrics/particle/momentum/all/assigned/samples": 35016.0, "metrics/particle/momentum/all/efficiency": 0.8075470924377441, "metrics/particle/momentum/all/event/energy_bias": 2.8436975479125977, "metrics/particle/momentum/all/event/p3_error_mean": 18.451034545898438, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.570244789123535, "metrics/particle/momentum/all/event/px_bias": 1.134897232055664, "metrics/particle/momentum/all/event/py_bias": -0.7828948497772217, "metrics/particle/momentum/all/event/pz_abs_mean": 13.54065990447998, "metrics/particle/momentum/all/event/pz_bias": 1.2297728061676025, "metrics/particle/momentum/all/event/pz_residual_rms": 24.384122848510742, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014449270442128181, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05516786873340607, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04776644706726074, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.014822571538388729, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0970323234796524, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018286146223545074, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17316538095474243, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.024120111018419266, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9794162511825562, "metrics/particle/momentum/class_1/accepted/samples": 15859.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10188291221857071, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.31659135222435, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3637000322341919, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3038860857486725, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05466744303703308, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.762519359588623, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0065927654504776, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.37620294094085693, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01747117191553116, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6477465629577637, "metrics/particle/momentum/class_1/assigned/samples": 17140.0, "metrics/particle/momentum/class_1/efficiency": 0.9251006245613098, "metrics/particle/momentum/class_1/event/energy_bias": 3.55092191696167, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.338584899902344, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.112360954284668, "metrics/particle/momentum/class_1/event/px_bias": 0.6205668449401855, "metrics/particle/momentum/class_1/event/py_bias": -0.22561606764793396, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.054670333862305, "metrics/particle/momentum/class_1/event/pz_bias": -0.9089512825012207, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.681528091430664, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04442260041832924, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20459817349910736, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20515766739845276, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05981789901852608, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24495700001716614, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04295532777905464, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.81320321559906, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04425473511219025, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.092844486236572, "metrics/particle/momentum/class_2/accepted/samples": 1164.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0079294443130493, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9222403764724731, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 14756.6982421875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 14756.185546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4011406898498535, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 756706.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08897338062524796, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4176688194274902, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1715579479932785, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.226125717163086, "metrics/particle/momentum/class_2/assigned/samples": 2630.0, "metrics/particle/momentum/class_2/efficiency": 0.40276816487312317, "metrics/particle/momentum/class_2/event/energy_bias": -6.944980621337891, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.643317222595215, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.816390991210938, "metrics/particle/momentum/class_2/event/px_bias": 0.6991268396377563, "metrics/particle/momentum/class_2/event/py_bias": 0.08995257318019867, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.959156036376953, "metrics/particle/momentum/class_2/event/pz_bias": -0.6826865673065186, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.01852035522461, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03207622468471527, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14064668118953705, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14073817431926727, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03377411141991615, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18259276449680328, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006149521213956177, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1896561086177826, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.05573713406920433, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.5989023447036743, "metrics/particle/momentum/class_3/accepted/samples": 11383.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.34866470098495483, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9133507013320923, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.964590311050415, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8280010223388672, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17806915938854218, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.93067169189453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010802363976836205, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5988205671310425, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.05177413299679756, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.6977641582489014, "metrics/particle/momentum/class_3/assigned/samples": 14719.0, "metrics/particle/momentum/class_3/efficiency": 0.7494239211082458, "metrics/particle/momentum/class_3/event/energy_bias": 9.10090446472168, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.703084945678711, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.683166980743408, "metrics/particle/momentum/class_3/event/px_bias": -0.20577119290828705, "metrics/particle/momentum/class_3/event/py_bias": -0.776821494102478, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.156627655029297, "metrics/particle/momentum/class_3/event/pz_bias": 2.400524377822876, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.387344360351562, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.019928142428398132, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18015222251415253, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11236526072025299, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.036663953214883804, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16731873154640198, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010067113675177097, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6456262469291687, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.30460214614868164, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.385711908340454, "metrics/particle/momentum/class_4/accepted/samples": 298.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.19271226227283478, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.621148943901062, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4278314113616943, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.345201015472412, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12711864709854126, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.648563385009766, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6649579405784607, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2401401549577713, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2620341777801514, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8418079018592834, "metrics/particle/momentum/class_4/event/energy_bias": -2.5911214351654053, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.947500705718994, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1085963249206543, "metrics/particle/momentum/class_4/event/px_bias": 0.06683281809091568, "metrics/particle/momentum/class_4/event/py_bias": 0.091530442237854, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6919773817062378, "metrics/particle/momentum/class_4/event/pz_bias": 0.41682517528533936, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.396059989929199, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015092750079929829, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.036037053912878036, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03417690843343735, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01089877262711525, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08702956140041351, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3376908302307129, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.14501354098320007, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8743507862091064, "metrics/particle/momentum/class_5/accepted/samples": 165.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.035741958767175674, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07296039164066315, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.07558730244636536, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.033243145793676376, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.040462426841259, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.24780906736850739, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.3870563805103302, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1293981522321701, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.9353631734848022, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9537572264671326, "metrics/particle/momentum/class_5/event/energy_bias": -0.272027850151062, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.3193100690841675, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9887855052947998, "metrics/particle/momentum/class_5/event/px_bias": -0.045858219265937805, "metrics/particle/momentum/class_5/event/py_bias": 0.03805974870920181, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.702529788017273, "metrics/particle/momentum/class_5/event/pz_bias": 0.00406096875667572, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.4893598556518555, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.779522180557251, "step_valid_time": 2671.007947921753, "step_total_time": 2672.7874703407288, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0055703653506862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10543250315625241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2859, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9445356087955366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9470220467258966, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9457771935589878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052977953274103325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9382999671808336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9407699901283317, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9395333552415379, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9445356087955366, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0136037115835592, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11708180153932435, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2264, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2133, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9363476733977173, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9421378091872792, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9392338177014531, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05786219081272085, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9319578577699736, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9377208480565371, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9348304711580802, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9363476733977173, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0156300935179072, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11042239353881023, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1316, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1207, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1194, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9157814871016692, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9171732522796353, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9164768413059985, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08282674772036475, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9059180576631259, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9072948328267477, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9066059225512528, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9157814871016692} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_7000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..ee5ee772a7f99f02043a4aca27baec75f8638886 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.33156498673740054, "train": {"Classification_binary": 3.0864295959472656, "Classification": 0.021196464076638222, "Regression_pt": 0.18527913093566895, "Regression_eta": 0.01070080604404211, "Regression_sin_phi": 0.008948246948421001, "Regression_cos_phi": 0.009321054443717003, "Regression_energy": 0.18419183790683746, "Total": 10.057546615600586, "Total_unweighted": 3.5060671363025904}, "train_group": {}, "valid": {"Classification_binary": 3.054673512776693, "Classification": 0.022085095445315044, "Regression_pt": 0.1665783723195394, "Regression_eta": 0.013207783301671347, "Regression_sin_phi": 0.007565877089897792, "Regression_cos_phi": 0.008106190711259842, "Regression_energy": 0.17234545946121216, "Total": 9.663340250651041, "Total_unweighted": 3.4445622911055884, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.022141186520457268, "diagnostic/loss/cld_pf/Classification_binary": 3.037229061126709, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007605080958455801, "diagnostic/loss/cld_pf/Regression_energy": 0.17197856307029724, "diagnostic/loss/cld_pf/Regression_eta": 0.01219103578478098, "diagnostic/loss/cld_pf/Regression_pt": 0.16721829771995544, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.007063734345138073, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08795920759439468, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21329236030578613, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06083992123603821, "diagnostic/regression/cld_pf/energy_residual_rms": 0.46508488059043884, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04097145423293114, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20898672938346863, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06813859939575195, "diagnostic/regression/cld_pf/pt_residual_rms": 0.45721080899238586, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9255625009536743, "metrics/particle/class_1/pid_accuracy": 0.9835590124130249, "metrics/particle/class_2/efficiency": 0.3812236487865448, "metrics/particle/class_2/pid_accuracy": 0.6521219611167908, "metrics/particle/class_3/efficiency": 0.7514284253120422, "metrics/particle/class_3/pid_accuracy": 0.9807270765304565, "metrics/particle/class_4/efficiency": 0.8395559787750244, "metrics/particle/class_4/pid_accuracy": 0.22956730425357819, "metrics/particle/class_5/efficiency": 0.9631068110466003, "metrics/particle/class_5/pid_accuracy": 0.5443548560142517, "metrics/particle/count/bias_mean": -0.14640523493289948, "metrics/particle/count/mae": 3.6392157077789307, "metrics/particle/count/prediction_mean": 70.41895294189453, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06054777279496193, "metrics/particle/event/energy_response_mean": 0.9943485260009766, "metrics/particle/event/met_abs_error": 7.192737579345703, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05119962617754936, "metrics/particle/event/scalar_pt_response_mean": 0.9988855719566345, "metrics/particle/event/vector_pt_closure": 0.05709271505475044, "metrics/particle/matched/delta_eta_abs_mean": 0.011578595265746117, "metrics/particle/matched/delta_phi_abs_mean": 0.013337166048586369, "metrics/particle/matched/delta_r_mean": 0.01965360715985298, "metrics/particle/matched/energy_relative_abs_mean": 0.08979815244674683, "metrics/particle/matched/pid_accuracy": 0.9600279927253723, "metrics/particle/matched/pt_relative_abs_mean": 0.08580739796161652, "metrics/particle/matching/duplicate_fraction": 0.03821200877428055, "metrics/particle/matching/efficiency": 0.8073078989982605, "metrics/particle/matching/f1": 0.8081462979316711, "metrics/particle/matching/purity": 0.8089863657951355, "metrics/particle/matching/target_coverage_dr0p05": 0.7599592208862305, "metrics/particle/matching/target_coverage_dr0p10": 0.8431991934776306, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7040707468986511, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8480803966522217, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022310784086585045, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08550120890140533, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08327987790107727, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00909634493291378, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1355791985988617, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00526976166293025, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3722987771034241, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.022650113329291344, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5828709602355957, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7970.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3203681707382202, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2962902784347534, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2286673784255981, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1019877195358276, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13877882063388824, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.853734970092773, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020623793825507164, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9127296805381775, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02746572718024254, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3172974586486816, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9843.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7862286567687988, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.45855483412742615, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.946016311645508, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.040027618408203, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.7335346341133118, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2486727386713028, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.7005791664123535, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.2486886978149414, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.457817077636719, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03304179012775421, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09498408436775208, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09462971985340118, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013429026119410992, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14647874236106873, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024088094010949135, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0093741416931152, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3069513440132141, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.441397190093994, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1453.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8769857883453369, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6661958694458008, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12744.490234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12744.3193359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2931978106498718, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 644399.1875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0895230621099472, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.29656720161438, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.27960920333862305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.625126838684082, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2558.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5285558104515076, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.216503381729126, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.7294745445251465, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4038751125335693, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02514592930674553, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.13998252153396606, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.535750389099121, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.1995781660079956, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.547887802124023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017716791480779648, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09392420947551727, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08894450217485428, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007642408367246389, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14502795040607452, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005631207022815943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12142915278673172, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006681915372610092, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5356029868125916, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19534.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17303963005542755, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.824415922164917, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5773460268974304, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4464096128940582, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1036200299859047, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.485789775848389, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004180378746241331, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.328055202960968, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0031785890460014343, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.8177950382232666, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22486.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8543935418128967, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.3181092739105225, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.012517929077148, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.448329448699951, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.2699449062347412, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7672858238220215, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.217251777648926, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.002881646156311, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.83602237701416, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01975019834935665, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09165908396244049, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08767066150903702, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008332943543791771, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14256428182125092, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030389889143407345, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2350325882434845, "metrics/particle/momentum/all/accepted/pz_bias": -0.026143807917833328, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2154767513275146, "metrics/particle/momentum/all/accepted/samples": 28957.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26622188091278076, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.019271731376648, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 935.1759033203125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 935.0432739257812, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12744002044200897, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 174491.15625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015077249146997929, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7106735110282898, "metrics/particle/momentum/all/assigned/pz_bias": -0.014801187440752983, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.874948024749756, "metrics/particle/momentum/all/assigned/samples": 34887.0, "metrics/particle/momentum/all/efficiency": 0.8100086450576782, "metrics/particle/momentum/all/event/energy_bias": 0.35694900155067444, "metrics/particle/momentum/all/event/p3_error_mean": 18.47553253173828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.455324172973633, "metrics/particle/momentum/all/event/px_bias": 1.9783337116241455, "metrics/particle/momentum/all/event/py_bias": 1.155941128730774, "metrics/particle/momentum/all/event/pz_abs_mean": 13.64651870727539, "metrics/particle/momentum/all/event/pz_bias": -1.451148509979248, "metrics/particle/momentum/all/event/pz_residual_rms": 27.170501708984375, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010464669205248356, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04986412450671196, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04356545954942703, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0033519279677420855, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09545356780290604, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001826886786147952, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15269671380519867, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.016723409295082092, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8825798630714417, "metrics/particle/momentum/class_1/accepted/samples": 15874.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0974762961268425, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3653793931007385, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.39933687448501587, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3350260853767395, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.054673824459314346, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.308244228363037, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007235383149236441, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.37269094586372375, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01655399240553379, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.794468879699707, "metrics/particle/momentum/class_1/assigned/samples": 17138.0, "metrics/particle/momentum/class_1/efficiency": 0.9259756207466125, "metrics/particle/momentum/class_1/event/energy_bias": 5.20927095413208, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.039887428283691, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.466151237487793, "metrics/particle/momentum/class_1/event/px_bias": 0.3342331647872925, "metrics/particle/momentum/class_1/event/py_bias": 0.07730983197689056, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.499126434326172, "metrics/particle/momentum/class_1/event/pz_bias": -1.0128716230392456, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.710840225219727, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04166927561163902, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20144082605838776, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20117974281311035, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0651271641254425, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24064265191555023, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0457809679210186, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9227255582809448, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3174132704734802, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.477650165557861, "metrics/particle/momentum/class_2/accepted/samples": 1114.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0330308675765991, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9329206943511963, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12584.9775390625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12584.4443359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.41389960050582886, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 640405.9375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08957529067993164, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.579136610031128, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3195432722568512, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.357531547546387, "metrics/particle/momentum/class_2/assigned/samples": 2590.0, "metrics/particle/momentum/class_2/efficiency": 0.38546714186668396, "metrics/particle/momentum/class_2/event/energy_bias": -9.099444389343262, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.26179027557373, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.422828674316406, "metrics/particle/momentum/class_2/event/px_bias": 0.5788721442222595, "metrics/particle/momentum/class_2/event/py_bias": 0.8715624213218689, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.842366218566895, "metrics/particle/momentum/class_2/event/pz_bias": -0.5377411842346191, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.923580169677734, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0307075884193182, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13765627145767212, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1378655731678009, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02149410918354988, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17958366870880127, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006082725012674928, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.180999293923378, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00993320532143116, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8119311332702637, "metrics/particle/momentum/class_3/accepted/samples": 11508.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3316192030906677, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7643903493881226, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.541864275932312, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3976600170135498, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16285109519958496, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.181703567504883, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011344222351908684, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6029367446899414, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.037296559661626816, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.302515745162964, "metrics/particle/momentum/class_3/assigned/samples": 14633.0, "metrics/particle/momentum/class_3/efficiency": 0.7576535940170288, "metrics/particle/momentum/class_3/event/energy_bias": 6.303508758544922, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.727547645568848, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.990323066711426, "metrics/particle/momentum/class_3/event/px_bias": 0.8114852905273438, "metrics/particle/momentum/class_3/event/py_bias": 0.1128503829240799, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.612322807312012, "metrics/particle/momentum/class_3/event/pz_bias": -0.10568501800298691, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.301098823547363, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014757060445845127, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.161172017455101, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.10086383670568466, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03936399146914482, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.14756372570991516, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003401360474526882, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3916661739349365, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.031854428350925446, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1480438709259033, "metrics/particle/momentum/class_4/accepted/samples": 294.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2376248985528946, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.975670576095581, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.754019260406494, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.6810123920440674, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14164306223392487, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.278648376464844, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01133144460618496, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7546244859695435, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.17157624661922455, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.151489496231079, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8305084705352783, "metrics/particle/momentum/class_4/event/energy_bias": -1.8257384300231934, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8547983169555664, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9548957347869873, "metrics/particle/momentum/class_4/event/px_bias": 0.042213380336761475, "metrics/particle/momentum/class_4/event/py_bias": 0.19732388854026794, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7186157703399658, "metrics/particle/momentum/class_4/event/pz_bias": 0.1438964307308197, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.4471211433410645, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009875595569610596, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.040066909044981, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.04069577902555466, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.010257350280880928, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.10541488230228424, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.25104978680610657, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08565232157707214, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7992459535598755, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.029585516080260277, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07201898843050003, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06928377598524094, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.023193564265966415, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.023121386766433716, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.22641582787036896, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.27150169014930725, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.06576018035411835, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.807611882686615, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.23064777255058289, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1335400342941284, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8824653625488281, "metrics/particle/momentum/class_5/event/px_bias": 0.2115296721458435, "metrics/particle/momentum/class_5/event/py_bias": -0.1031053364276886, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5555391907691956, "metrics/particle/momentum/class_5/event/pz_bias": 0.0612528994679451, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.28372859954834, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.451026201248169, "step_valid_time": 3114.176518201828, "step_total_time": 3115.6275446414948, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9940036815329505, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10045628808104456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2884, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2865, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.946504758779127, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9502471169686986, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9483722459717199, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04975288303130148, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9402691171644241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.943986820428336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.942124301216705, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.946504758779127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9962278107391287, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11026030708025203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2266, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9306409130816505, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9355692850838482, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9330985915492958, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06443071491615181, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.92756804214223, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9324801412180053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9300176056338029, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9306409130816505, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018008752351013, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11319533726949982, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1206, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1200, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9150227617602428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9178082191780822, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9164133738601824, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0821917808219178, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9104704097116844, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.91324200913242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9118541033434651, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9150227617602428} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_8000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..a6807806e0837e12fd110d3cb7f4e0f7da40877f --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.378931413414172, "train": {"Classification_binary": 3.02023983001709, "Classification": 0.02263709530234337, "Regression_pt": 0.15838667750358582, "Regression_eta": 0.010667318478226662, "Regression_sin_phi": 0.006917676888406277, "Regression_cos_phi": 0.009819635190069675, "Regression_energy": 0.16024644672870636, "Total": 9.378496170043945, "Total_unweighted": 3.388914680108428}, "train_group": {}, "valid": {"Classification_binary": 3.0132048924764, "Classification": 0.021899526317914326, "Regression_pt": 0.16592568159103394, "Regression_eta": 0.013090141117572784, "Regression_sin_phi": 0.007391002650062243, "Regression_cos_phi": 0.007999049499630928, "Regression_energy": 0.17142788569132486, "Total": 9.571004867553711, "Total_unweighted": 3.400938179343939, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.022328104823827744, "diagnostic/loss/cld_pf/Classification_binary": 3.008011817932129, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007601108402013779, "diagnostic/loss/cld_pf/Regression_energy": 0.1712796688079834, "diagnostic/loss/cld_pf/Regression_eta": 0.011900261975824833, "diagnostic/loss/cld_pf/Regression_pt": 0.16730378568172455, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006884027272462845, "diagnostic/regression/cld_pf/energy_pred_mean": 0.08003108948469162, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.2068692296743393, "diagnostic/regression/cld_pf/energy_residual_mean": 0.05291181057691574, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4546809196472168, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.03444116935133934, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20310090482234955, "diagnostic/regression/cld_pf/pt_residual_mean": 0.06160830706357956, "diagnostic/regression/cld_pf/pt_residual_rms": 0.44917017221450806, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9259458184242249, "metrics/particle/class_1/pid_accuracy": 0.9837314486503601, "metrics/particle/class_2/efficiency": 0.3708556592464447, "metrics/particle/class_2/pid_accuracy": 0.6758832335472107, "metrics/particle/class_3/efficiency": 0.7554720640182495, "metrics/particle/class_3/pid_accuracy": 0.9810338020324707, "metrics/particle/class_4/efficiency": 0.8657921552658081, "metrics/particle/class_4/pid_accuracy": 0.2226107269525528, "metrics/particle/class_5/efficiency": 0.9747572541236877, "metrics/particle/class_5/pid_accuracy": 0.5318725109100342, "metrics/particle/count/bias_mean": -1.317647099494934, "metrics/particle/count/mae": 3.8379085063934326, "metrics/particle/count/prediction_mean": 69.24771118164062, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.06355404108762741, "metrics/particle/event/energy_response_mean": 0.9698835015296936, "metrics/particle/event/met_abs_error": 6.838744640350342, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050850994884967804, "metrics/particle/event/scalar_pt_response_mean": 0.9837750792503357, "metrics/particle/event/vector_pt_closure": 0.057636111974716187, "metrics/particle/matched/delta_eta_abs_mean": 0.01290097925812006, "metrics/particle/matched/delta_phi_abs_mean": 0.01281249150633812, "metrics/particle/matched/delta_r_mean": 0.02029152773320675, "metrics/particle/matched/energy_relative_abs_mean": 0.08926095068454742, "metrics/particle/matched/pid_accuracy": 0.9611138105392456, "metrics/particle/matched/pt_relative_abs_mean": 0.08504593372344971, "metrics/particle/matching/duplicate_fraction": 0.033931419253349304, "metrics/particle/matching/efficiency": 0.8086509704589844, "metrics/particle/matching/f1": 0.816271960735321, "metrics/particle/matching/purity": 0.8240379691123962, "metrics/particle/matching/target_coverage_dr0p05": 0.7631176710128784, "metrics/particle/matching/target_coverage_dr0p10": 0.8416523933410645, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7075996994972229, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8482378721237183, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02251855842769146, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08450011909008026, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08216111361980438, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012705937959253788, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13355329632759094, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004987531341612339, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37513235211372375, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.027394121512770653, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6241284608840942, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8020.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.302632212638855, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1742278337478638, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2556962966918945, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1319396495819092, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1219940111041069, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.452308654785156, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016410361975431442, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8140971064567566, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06573435664176941, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.1696858406066895, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9689.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7911611199378967, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.039628505706787, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.41894245147705, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.639244318008423, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.08511818945407867, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.18627390265464783, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.2862229347229, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.7582991123199463, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.68578052520752, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029837049543857574, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0878773182630539, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08837684988975525, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016863055527210236, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13839416205883026, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01982228271663189, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8925891518592834, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.22376060485839844, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1192474365234375, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1463.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8366572856903076, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.495162010192871, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 30224.30859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 30224.130859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28577154874801636, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1509543.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08496993780136108, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.233792781829834, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.40871870517730713, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.110892295837402, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2495.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5321935415267944, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.309432506561279, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.441097259521484, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2660845518112183, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0635891705751419, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06716560572385788, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.268984794616699, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5283825397491455, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.665504455566406, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018471889197826385, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09271480143070221, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08708305656909943, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012144210748374462, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1429390162229538, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005658727022819221, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12044823914766312, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02374722622334957, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5245562791824341, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19439.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17044617235660553, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6401399374008179, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5022448897361755, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3763841390609741, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10063682496547699, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.760484218597412, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034980715718120337, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2986975312232971, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.030423535034060478, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6400288343429565, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22298.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8502383828163147, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.2102930545806885, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.488897323608398, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.097671985626221, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5576238036155701, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.32427579164505005, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.9295153617858887, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.3031275272369385, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.025511741638184, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020168917253613472, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09019219130277634, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08578365296125412, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012538676150143147, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14016874134540558, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002766060410067439, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23012974858283997, "metrics/particle/momentum/all/accepted/pz_bias": -0.03487604111433029, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1868293285369873, "metrics/particle/momentum/all/accepted/samples": 28922.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2557935416698456, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8520782589912415, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2187.605712890625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2187.4765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1200336441397667, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 406054.9375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013021286576986313, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6558916568756104, "metrics/particle/momentum/all/assigned/pz_bias": -0.06771758198738098, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4582343101501465, "metrics/particle/momentum/all/assigned/samples": 34482.0, "metrics/particle/momentum/all/efficiency": 0.8090296387672424, "metrics/particle/momentum/all/event/energy_bias": -8.55935287475586, "metrics/particle/momentum/all/event/p3_error_mean": 17.904958724975586, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.253747940063477, "metrics/particle/momentum/all/event/px_bias": 0.5791528224945068, "metrics/particle/momentum/all/event/py_bias": -0.5777153372764587, "metrics/particle/momentum/all/event/pz_abs_mean": 13.244305610656738, "metrics/particle/momentum/all/event/pz_bias": -4.589808940887451, "metrics/particle/momentum/all/event/pz_residual_rms": 22.640182495117188, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011732812970876694, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04895934835076332, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.041643448173999786, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008133316412568092, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09160060435533524, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001768569927662611, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.148171529173851, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03327685594558716, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8359962701797485, "metrics/particle/momentum/class_1/accepted/samples": 15832.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09974353760480881, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24826335906982422, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3595448434352875, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.29437384009361267, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.055273447185754776, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.911920547485352, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006770559586584568, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.38319557905197144, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.050221603363752365, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.0522658824920654, "metrics/particle/momentum/class_1/assigned/samples": 17133.0, "metrics/particle/momentum/class_1/efficiency": 0.9235256314277649, "metrics/particle/momentum/class_1/event/energy_bias": 1.3948708772659302, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.546026229858398, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.44262409210205, "metrics/particle/momentum/class_1/event/px_bias": 0.5755340456962585, "metrics/particle/momentum/class_1/event/py_bias": -0.363999605178833, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.86749267578125, "metrics/particle/momentum/class_1/event/pz_bias": -1.5856735706329346, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.522003173828125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.040771301835775375, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19824856519699097, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1987033486366272, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.055208589881658554, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23777461051940918, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03963133692741394, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8967769145965576, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.19941681623458862, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.465612888336182, "metrics/particle/momentum/class_2/accepted/samples": 1085.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0003564357757568, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7583386301994324, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 30577.53125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 30577.0, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4124087691307068, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1518394.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0847526341676712, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4313735961914062, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.47170311212539673, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.850639343261719, "metrics/particle/momentum/class_2/assigned/samples": 2466.0, "metrics/particle/momentum/class_2/efficiency": 0.3754325211048126, "metrics/particle/momentum/class_2/event/energy_bias": -12.684194564819336, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.089212417602539, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.473649978637695, "metrics/particle/momentum/class_2/event/px_bias": -0.2466254085302353, "metrics/particle/momentum/class_2/event/py_bias": -0.07396458834409714, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.503669738769531, "metrics/particle/momentum/class_2/event/pz_bias": -2.9771976470947266, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.326074600219727, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030083006247878075, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13482005894184113, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13573364913463593, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023745516315102577, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17775042355060577, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006072171963751316, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.18065477907657623, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.021845871582627296, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.80165696144104, "metrics/particle/momentum/class_3/accepted/samples": 11528.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3188357651233673, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5595076084136963, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5213500261306763, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.382532000541687, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14870794117450714, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.816208839416504, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008358292281627655, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5101135969161987, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.019054559990763664, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.623649835586548, "metrics/particle/momentum/class_3/assigned/samples": 14357.0, "metrics/particle/momentum/class_3/efficiency": 0.758970320224762, "metrics/particle/momentum/class_3/event/energy_bias": 5.081225395202637, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.256004333496094, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.895172119140625, "metrics/particle/momentum/class_3/event/px_bias": 0.07235535979270935, "metrics/particle/momentum/class_3/event/py_bias": -0.33845049142837524, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.169357776641846, "metrics/particle/momentum/class_3/event/pz_bias": -0.6376291513442993, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.54166030883789, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01589233987033367, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1919599026441574, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11905083060264587, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06022508814930916, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1705985963344574, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006451612804085016, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.4637027978897095, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.030293932184576988, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3856397867202759, "metrics/particle/momentum/class_4/accepted/samples": 310.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.17091168463230133, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4350318908691406, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3141355514526367, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.255190849304199, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0991501435637474, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.981094360351562, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008498583920300007, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6127631664276123, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.1314249038696289, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1833646297454834, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.8757061958312988, "metrics/particle/momentum/class_4/event/energy_bias": -2.0884437561035156, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6896893978118896, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9372754096984863, "metrics/particle/momentum/class_4/event/px_bias": 0.018943630158901215, "metrics/particle/momentum/class_4/event/py_bias": 0.23521971702575684, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5323156118392944, "metrics/particle/momentum/class_4/event/pz_bias": 0.4496276378631592, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.211668014526367, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009647728875279427, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02755202166736126, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026943229138851166, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008204462938010693, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06935068219900131, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15342305600643158, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.025438329204916954, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.44586044549942017, "metrics/particle/momentum/class_5/accepted/samples": 167.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03832617774605751, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04847104847431183, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04912848398089409, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.00809747725725174, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.028901733458042145, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14648814499378204, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.28551003336906433, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.04964439198374748, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 1.2522865533828735, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9653179049491882, "metrics/particle/momentum/class_5/event/energy_bias": -0.2628118395805359, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0858491659164429, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8494405150413513, "metrics/particle/momentum/class_5/event/px_bias": 0.15894518792629242, "metrics/particle/momentum/class_5/event/py_bias": -0.03652035444974899, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.539730966091156, "metrics/particle/momentum/class_5/event/pz_bias": 0.16106340289115906, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.2128043174743652, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.4348998069763184, "step_valid_time": 3559.9149663448334, "step_total_time": 3561.3498663902283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001461150692295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10623498118334396, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2884, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2869, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.946504758779127, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9486842105263158, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.947593231476918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05131578947368421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.941581883820151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.94375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9426646952521768, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.946504758779127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9932156130960293, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11124001705115649, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2137, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9416154521510096, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9436867575890893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.942649967040211, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056313242410910694, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9381035996488147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9401671799384074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.939134256207427, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9416154521510096, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9941279252812267, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1067215410377178, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1211, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9188163884673748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9223153084539223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9205625237552262, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07768469154607768, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.917298937784522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9207920792079208, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9190421892816419, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9188163884673748} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_9000.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..98827bdd5bff83b5ecde31f7d4af12926790de4a --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.4262978400909435, "train": {"Classification_binary": 2.8525164127349854, "Classification": 0.022128582000732422, "Regression_pt": 0.15015926957130432, "Regression_eta": 0.008479506708681583, "Regression_sin_phi": 0.0063081542029976845, "Regression_cos_phi": 0.007208158262073994, "Regression_energy": 0.15176892280578613, "Total": 8.66571044921875, "Total_unweighted": 3.1985690062865615}, "train_group": {}, "valid": {"Classification_binary": 2.9784278869628906, "Classification": 0.021962563196818035, "Regression_pt": 0.16272051135698953, "Regression_eta": 0.013979649792114893, "Regression_sin_phi": 0.007483534514904022, "Regression_cos_phi": 0.008024719854195913, "Regression_energy": 0.1677756905555725, "Total": 9.512083053588867, "Total_unweighted": 3.3603745562334857, "diagnostic/composition/cld_pf/events": 128.0, "diagnostic/composition/cld_pf/target_particles": 8937.25, "diagnostic/composition/cld_pf/valid_elements": 14339.75, "diagnostic/loss/cld_pf/Classification": 0.02202274464070797, "diagnostic/loss/cld_pf/Classification_binary": 2.9708633422851562, "diagnostic/loss/cld_pf/Regression_cos_phi": 0.007703546434640884, "diagnostic/loss/cld_pf/Regression_energy": 0.16740886867046356, "diagnostic/loss/cld_pf/Regression_eta": 0.012979223392903805, "diagnostic/loss/cld_pf/Regression_pt": 0.1635536551475525, "diagnostic/loss/cld_pf/Regression_sin_phi": 0.006923099048435688, "diagnostic/regression/cld_pf/energy_pred_mean": 0.09383419901132584, "diagnostic/regression/cld_pf/energy_residual_abs_mean": 0.21066221594810486, "diagnostic/regression/cld_pf/energy_residual_mean": 0.06671492010354996, "diagnostic/regression/cld_pf/energy_residual_rms": 0.4622853398323059, "diagnostic/regression/cld_pf/energy_target_mean": 0.027119280770421028, "diagnostic/regression/cld_pf/pt_pred_mean": 0.04680866748094559, "diagnostic/regression/cld_pf/pt_residual_abs_mean": 0.20626473426818848, "diagnostic/regression/cld_pf/pt_residual_mean": 0.07397580146789551, "diagnostic/regression/cld_pf/pt_residual_rms": 0.4553541839122772, "diagnostic/regression/cld_pf/pt_target_mean": -0.02716713957488537, "metrics/particle/class_1/efficiency": 0.9328261017799377, "metrics/particle/class_1/pid_accuracy": 0.9826392531394958, "metrics/particle/class_2/efficiency": 0.3809957802295685, "metrics/particle/class_2/pid_accuracy": 0.6800239086151123, "metrics/particle/class_3/efficiency": 0.7376494407653809, "metrics/particle/class_3/pid_accuracy": 0.9741702675819397, "metrics/particle/class_4/efficiency": 0.8506559133529663, "metrics/particle/class_4/pid_accuracy": 0.2526690363883972, "metrics/particle/class_5/efficiency": 0.9766990542411804, "metrics/particle/class_5/pid_accuracy": 0.5049701929092407, "metrics/particle/count/bias_mean": 0.6431372761726379, "metrics/particle/count/mae": 3.644444465637207, "metrics/particle/count/prediction_mean": 71.20849609375, "metrics/particle/count/target_mean": 70.56536102294922, "metrics/particle/event/energy_relative_abs_error": 0.05747124180197716, "metrics/particle/event/energy_response_mean": 0.9990694522857666, "metrics/particle/event/met_abs_error": 6.332637786865234, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05011774227023125, "metrics/particle/event/scalar_pt_response_mean": 1.008725643157959, "metrics/particle/event/vector_pt_closure": 0.05583273619413376, "metrics/particle/matched/delta_eta_abs_mean": 0.022995274513959885, "metrics/particle/matched/delta_phi_abs_mean": 0.015391990542411804, "metrics/particle/matched/delta_r_mean": 0.030016835778951645, "metrics/particle/matched/energy_relative_abs_mean": 0.09003044664859772, "metrics/particle/matched/pid_accuracy": 0.9578851461410522, "metrics/particle/matched/pt_relative_abs_mean": 0.08609923720359802, "metrics/particle/matching/duplicate_fraction": 0.03963322192430496, "metrics/particle/matching/efficiency": 0.8051590919494629, "metrics/particle/matching/f1": 0.8015065789222717, "metrics/particle/matching/purity": 0.7978870868682861, "metrics/particle/matching/target_coverage_dr0p05": 0.6659936308860779, "metrics/particle/matching/target_coverage_dr0p10": 0.8414300680160522, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7040337324142456, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8523780703544617, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03283851221203804, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08528019487857819, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0829576849937439, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00790745485574007, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13550038635730743, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004913076292723417, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.380854070186615, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05758493021130562, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5056884288787842, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7938.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33187630772590637, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.305346965789795, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.447689414024353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3220783472061157, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13681340217590332, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.141864776611328, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0199919231235981, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9018499255180359, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.021034687757492065, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4708940982818604, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9904.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7830719351768494, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7007724046707153, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.575204849243164, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.924476623535156, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1348668485879898, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09364645183086395, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.990761756896973, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.8820934295654297, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.598690032958984, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10137.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03589577227830887, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08961056172847748, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08949179202318192, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.00024088678765110672, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1392747163772583, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018493151292204857, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8476232290267944, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12492702156305313, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.6940064430236816, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1460.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8515251278877258, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7093530893325806, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 53245.078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 53244.8828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2947819232940674, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2698056.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0868380069732666, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1619713306427, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.007119452115148306, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.135836601257324, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2568.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5311022400856018, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7067880630493164, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.604270935058594, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3934309482574463, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01952553167939186, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05055425688624382, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.399471282958984, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.6639230251312256, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.725181579589844, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2749.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028481172397732735, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09316904097795486, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08813907951116562, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016275344416499138, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14440615475177765, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00077220075763762, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14706405997276306, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.07052566111087799, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5782949924468994, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19425.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18543772399425507, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7415754795074463, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6317890286445618, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5123748183250427, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10765688121318817, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.722845554351807, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0036313715390861034, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3252207934856415, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.05721791833639145, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.572314739227295, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22581.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8496260046958923, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5942793488502502, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.252942085266113, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.025745391845703, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.117020845413208, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.8780927658081055, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.4941253662109375, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6875886917114258, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.81493854522705, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 22863.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0300567839294672, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09081616252660751, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0867806226015091, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013134172186255455, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14174935221672058, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028102556243538857, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24693714082241058, "metrics/particle/momentum/all/accepted/pz_bias": 0.05706125870347023, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1033507585525513, "metrics/particle/momentum/all/accepted/samples": 28823.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.27561089396476746, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9717652797698975, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3901.57666015625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3901.44970703125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12960374355316162, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 730274.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014349699020385742, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6959653496742249, "metrics/particle/momentum/all/assigned/pz_bias": 0.043324340134859085, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.33331298828125, "metrics/particle/momentum/all/assigned/samples": 35053.0, "metrics/particle/momentum/all/efficiency": 0.8062602877616882, "metrics/particle/momentum/all/event/energy_bias": 0.5882636904716492, "metrics/particle/momentum/all/event/p3_error_mean": 18.18580436706543, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.513728141784668, "metrics/particle/momentum/all/event/px_bias": -2.2714133262634277, "metrics/particle/momentum/all/event/py_bias": -1.8350005149841309, "metrics/particle/momentum/all/event/pz_abs_mean": 13.447234153747559, "metrics/particle/momentum/all/event/pz_bias": 3.23360538482666, "metrics/particle/momentum/all/event/pz_residual_rms": 21.532114028930664, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35749.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017464732751250267, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04851849377155304, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.04158608987927437, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.001687456970103085, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.09217648208141327, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017550457268953323, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.17099107801914215, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.04474499449133873, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.856895923614502, "metrics/particle/momentum/class_1/accepted/samples": 15954.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09321421384811401, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2726174294948578, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.38015514612197876, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3156791925430298, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04987458512187004, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.594830513000488, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006708277389407158, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.36960935592651367, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.021582402288913727, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6450064182281494, "metrics/particle/momentum/class_1/assigned/samples": 17143.0, "metrics/particle/momentum/class_1/efficiency": 0.9306422472000122, "metrics/particle/momentum/class_1/event/energy_bias": 2.4744155406951904, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.17902660369873, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.12084674835205, "metrics/particle/momentum/class_1/event/px_bias": -0.5932504534721375, "metrics/particle/momentum/class_1/event/py_bias": -0.6720995903015137, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.694890975952148, "metrics/particle/momentum/class_1/event/pz_bias": 0.27086812257766724, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.507734298706055, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17143.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.05231023579835892, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19532226026058197, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19864395260810852, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.049680329859256744, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2385910153388977, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03726708143949509, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7549684047698975, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.019862784072756767, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.026304721832275, "metrics/particle/momentum/class_2/accepted/samples": 1127.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9979180693626404, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8358476758003235, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 51556.72265625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 51556.22265625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40761688351631165, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2654982.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.080693818628788, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.255338430404663, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2108456939458847, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.086735725402832, "metrics/particle/momentum/class_2/assigned/samples": 2652.0, "metrics/particle/momentum/class_2/efficiency": 0.38996538519859314, "metrics/particle/momentum/class_2/event/energy_bias": -6.595858573913574, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.606485366821289, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.17601490020752, "metrics/particle/momentum/class_2/event/px_bias": -1.0835087299346924, "metrics/particle/momentum/class_2/event/py_bias": -0.7025964260101318, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.27479076385498, "metrics/particle/momentum/class_2/event/pz_bias": 0.6333178877830505, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.242877960205078, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2890.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.04611827805638313, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13951413333415985, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13974377512931824, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03474317863583565, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18123795092105865, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008873114711605012, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19684891402721405, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.07451197504997253, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6236868500709534, "metrics/particle/momentum/class_3/accepted/samples": 11270.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36220335960388184, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7847378253936768, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7739723920822144, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6398619413375854, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17378318309783936, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.194751739501953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011540289036929607, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.618696928024292, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03255149722099304, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.015941619873047, "metrics/particle/momentum/class_3/assigned/samples": 14731.0, "metrics/particle/momentum/class_3/efficiency": 0.7419843077659607, "metrics/particle/momentum/class_3/event/energy_bias": 6.7464094161987305, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.261838912963867, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.838519096374512, "metrics/particle/momentum/class_3/event/px_bias": -0.607885479927063, "metrics/particle/momentum/class_3/event/py_bias": -0.4980321228504181, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.237444877624512, "metrics/particle/momentum/class_3/event/pz_bias": 2.085724353790283, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.626302719116211, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15189.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02051704004406929, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1535324901342392, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.11391559988260269, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.054165467619895935, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.16735950112342834, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0033112582750618458, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5070925354957581, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13572995364665985, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.499993920326233, "metrics/particle/momentum/class_4/accepted/samples": 302.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22050176560878754, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4735827445983887, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.5201499462127686, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.43918514251709, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12429378181695938, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.187092781066895, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7410783767700195, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.24948367476463318, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.5228359699249268, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8531073331832886, "metrics/particle/momentum/class_4/event/energy_bias": -1.5294601917266846, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.537970781326294, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8395906686782837, "metrics/particle/momentum/class_4/event/px_bias": -0.054413147270679474, "metrics/particle/momentum/class_4/event/py_bias": 0.049371082335710526, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4376990795135498, "metrics/particle/momentum/class_4/event/pz_bias": 0.08412417769432068, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.675790786743164, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 354.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016420850530266762, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.027710260823369026, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027215490117669106, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0016430970281362534, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0678177997469902, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.23528644442558289, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.16287778317928314, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.48973554372787476, "metrics/particle/momentum/class_5/accepted/samples": 170.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01654999330639839, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.037678807973861694, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.037098340690135956, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0006432863883674145, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.017341040074825287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10393450409173965, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.28866150975227356, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.12523038685321808, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 0.7843059301376343, "metrics/particle/momentum/class_5/assigned/samples": 173.0, "metrics/particle/momentum/class_5/efficiency": 0.9826589822769165, "metrics/particle/momentum/class_5/event/energy_bias": -0.5072427988052368, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9784400463104248, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.773730456829071, "metrics/particle/momentum/class_5/event/px_bias": 0.06764449924230576, "metrics/particle/momentum/class_5/event/py_bias": -0.011643431149423122, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.46449100971221924, "metrics/particle/momentum/class_5/event/pz_bias": 0.15957054495811462, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.0674631595611572, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 173.0}, "step_train_time": 1.5459229946136475, "step_valid_time": 4002.999360561371, "step_total_time": 4004.5452840328217, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017636648417063, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09790951391529334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3052, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2879, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9448638004594684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9433158584534731, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9440891949499918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056684141546526866, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9389563505086971, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9374180865006553, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9381865879652402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9448638004594684, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0069224727185921, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1056398518085393, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2278, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2107, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9306409130816505, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9335094671950682, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9320729830732029, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06649053280493175, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9249341527655839, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9277851166886834, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9263574411958673, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9306409130816505, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.012748291429478, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10146994164174594, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1187, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9006069802731411, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9012908124525436, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9009487666034156, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09870918754745633, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8968133535660091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.89749430523918, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8971537001897533, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9006069802731411} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/hyperparameters.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..e107b32c89c618c88add848975010ec14be95b7a --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 8639954, "dataset": "cld", "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "heptv2", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "gnnlsh": null, "attention": null, "litept": null, "hept": null, "heptv2": {"activation_checkpointing": true, "embedding_dim": 256, "width": 256, "num_convs": 6, "num_heads": 16, "dropout_ff": 0.0, "activation": "elu", "pe_type": "learned", "block_size": 128, "n_hashes": 3, "num_regions": 140, "deterministic_hashing": true, "num_w_per_dist": 10, "mlp_ratio": 4.0}}, "conv_type": "heptv2", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 128, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf", "cld_edm_ww_fullhad_pf", "cld_edm_qq_pf"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/model_kwargs.pkl b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e793acf8a6fb790259a54d0d1054208be97244f7 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9630a465891b2b9e7774c6212493d87ae56afa1238f059daff58af24aac811fc +size 4859 diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/particleflow_spec.yaml b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..28b2fb87abbbb496e4ddc8a426c42e554575235b Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..2a3fc6b6f667cd1d1543462713801ce21786cf20 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_eta.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cef0c34d58a520c0c0fa8a0565a2c8646cddb5ec Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..349a2ebe890f17f58a44b3a5c12252042b10a3bd Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c569102230086a08678ab2fec2d2146d28366479 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d952a4f2501cbf9314e6c8af85c3b4f9b4fd49c4 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_pt_log.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6f3afb412917ec0b8a52d16f693f57f149c2a5b0 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..fedf925f6f5ad4de63195f8bd9c471854e2b01c3 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..36845724f51a3780afcc0f11ecb6c7837de23965 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..7a1ea4c189196703ea26ecf8287104d7647be92f Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/jet_res_vstarget.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ba38eea9e5944442b777650117297dd57afcfb24 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..ba56b74f421099b14e7ac5dd0135dfe376e869f1 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_qq_pf/num_elements.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b9b8e80adebdb232797c3c2f39965d26f062acfa Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..15f04efea07c05870807f8bdedf82658eb186e91 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5215597892c47f605dbe38c93dfe2245a9288cd6 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..256f2d263520282f9fc6c512425b63a87cafef1d Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e1a6cbe212901b59ba7d81c03338170f8272ea67 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d15312effcd1b1f674487a4a5cd91c14469f928e Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f3ca51ee429b07eff89e4eb73933d2d17016837f Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..6b0fbec73e3140594ec7dd43525ffe5994a8d995 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ec5b6577b403291bded6bd4d53328b8b6d719829 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..e64339cedd691598c3a2d37fb4a4bbe88c074933 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9a5e08d68db22baddb08afdbebae268ada1f1fd0 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..a0eff4c2eee3c8854cdd1971afc10246e6d543fc Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dfb2cdd810e191f1dcd0f1eec143363bb348b7f7 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..7c9f9b40592fc72ed91e98a40b318bc2945ffd9a Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_eta.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8355031e5abb9359f7c8335c5a69fef8fbcb9322 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..bef1b7bc6f88deab6f451288e031ba54eaa8c24b Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..838a8d60577dbb27faf3bf35c9ed7afd72efda8a Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..b87a9902235b73a8805b1e80ec76f78be269df3c Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_pt_log.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d5cf71a679cb79943161ef0d4a7374746b66ca11 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..56d40fe36aaf79575100d8604827274fa72fc7cd Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e3178104a56be058d2c5e6af28650f439e6e4c2a Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..2e219965f2e10b302b62b9a3f91459511212ae25 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/jet_res_vstarget.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cf3be745892cf6355457b2d157ef1da4541de9e7 Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.pdf differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..4836501e18e89599703fe6544f810383185d576c Binary files /dev/null and b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/plots_step_50000/cld_edm_ww_fullhad_pf/num_elements.png differ diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/runs/train/events.out.tfevents.1793178753.nid005725.85735.0 b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/runs/train/events.out.tfevents.1793178753.nid005725.85735.0 new file mode 100644 index 0000000000000000000000000000000000000000..a0a528d808aa0b9f65f0f33444c630848a45cea6 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/runs/train/events.out.tfevents.1793178753.nid005725.85735.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05675302666a98fbaebe9d3bd4a026065398efc126c03507f98169b56df9df91 +size 16258058 diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/runs/valid/events.out.tfevents.1793178753.nid005725.85735.1 b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/runs/valid/events.out.tfevents.1793178753.nid005725.85735.1 new file mode 100644 index 0000000000000000000000000000000000000000..bf46459ee98202a71e1b14732f27435ce43c76e8 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/runs/valid/events.out.tfevents.1793178753.nid005725.85735.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9f0a6e7e928c91eca10a03e0f4a3aa9eb342fea1fd8c719c54eb981bc2b084 +size 28841459 diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/scenario-manifest.json b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4f1a09e70c45145b6fcae80fcab4cfe62a74fdcd --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/scenario-manifest.json @@ -0,0 +1,846 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239271-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239271-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "cld_pf_heptv2", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 128, + "gpu_batch_multiplier": 128, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "seed": 12345, + "gpu_batch_multiplier": 128 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "heptv2", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "gnnlsh": null, + "attention": null, + "litept": null, + "hept": null, + "heptv2": { + "activation_checkpointing": true, + "embedding_dim": 256, + "width": 256, + "num_convs": 6, + "num_heads": 16, + "dropout_ff": 0.0, + "activation": "elu", + "pe_type": "learned", + "block_size": 128, + "n_hashes": 3, + "num_regions": 140, + "deterministic_hashing": true, + "num_w_per_dist": 10, + "mlp_ratio": 4.0 + } + }, + "conv_type": "heptv2", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 128, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf", + "cld_edm_ww_fullhad_pf", + "cld_edm_qq_pf" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 1 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "256", + "--model.heptv2.width", + "256", + "--model.heptv2.num_convs", + "6", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "128", + "--model.heptv2.deterministic_hashing", + "true", + "--model.heptv2.activation_checkpointing", + "true", + "--seed", + "12345", + "--gpu_batch_multiplier", + "128" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" +} \ No newline at end of file diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train-config.yaml b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7150d8f84ea0a002ad7696b643f8ac5699d68dc4 --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train-config.yaml @@ -0,0 +1,276 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: heptv2 +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +- cld_edm_ww_fullhad_pf +- cld_edm_qq_pf +gpu_batch_multiplier: 128 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: null + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: + activation: elu + activation_checkpointing: true + block_size: 128 + deterministic_hashing: true + dropout_ff: 0.0 + embedding_dim: 256 + mlp_ratio: 4.0 + n_hashes: 3 + num_convs: 6 + num_heads: 16 + num_regions: 140 + num_w_per_dist: 10 + pe_type: learned + width: 256 + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: false + trainable: all + type: heptv2 +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + cld_edm_qq_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ww_fullhad_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + cld_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train.log b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train.log new file mode 100644 index 0000000000000000000000000000000000000000..22aaf360e7de34d3ec3c6ea2744547cbc383becd --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train.log @@ -0,0 +1,13931 @@ +rank0 [2026-10-28 11:12:24,874] INFO : Will use single-gpu: AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,875] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-10-28 11:12:24,881] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,882] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,882] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,882] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,882] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,918] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,919] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,971] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,979] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,986] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,993] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:25,001] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:25,008] INFO : Backbone initialization took 0.09s +rank0 [2026-10-28 11:12:25,008] INFO : backbone parameters: 7901376 +rank0 [2026-10-28 11:12:25,009] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:25,013] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:25,013] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:25,013] INFO : jagged_attention enabled=False +rank0 [2026-10-28 11:12:25,013] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-28 11:12:25,014] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:25,014] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:25,014] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:25,014] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:25,014] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:25,014] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:25,015] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,015] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,015] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:25,015] INFO : Total MLPF parameters: 8639954 +rank0 [2026-10-28 11:12:25,015] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:25,015] INFO : Instantiated model +rank0 [2026-10-28 11:12:25,016] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,241] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,243] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,355] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +205 nn_energy.nn.1.0.bias 256 0 +206 nn_energy.nn.1.2.weight 256 0 +207 nn_energy.nn.1.2.bias 256 0 +208 nn_energy.nn.1.4.weight 256 0 +209 nn_energy.nn.1.4.bias 1 0 + +[210 rows x 3 columns] +rank0 [2026-10-28 11:12:25,356] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,357] INFO : Trainable parameters: 8639954 +rank0 [2026-10-28 11:12:25,357] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-28 11:12:25,357] INFO : Total parameters: 8663222 +rank0 [2026-10-28 11:12:25,360] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.regions 0 96 + backbone.0.attn.out_linear.weight 65536 0 + backbone.0.attn.out_linear.bias 256 0 + backbone.0.attn.e2lsh.alpha 0 864 + backbone.0.attn.e2lsh.beta 0 3 + backbone.0.attn.e2lsh_new.alpha 0 96 + backbone.0.attn.e2lsh_new.beta 0 3 + backbone.0.w_q.weight 65536 0 + backbone.0.w_k.weight 65536 0 + backbone.0.w_v.weight 65536 0 + backbone.0.w_rpe.weight 0 2560 + backbone.0.w_rpe.bias 0 256 +backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + backbone.0.pe_func.position_embedding_head.0.bias 1024 0 +backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + backbone.0.pe_func.position_embedding_head.1.bias 1024 0 +backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + backbone.0.pe_func.position_embedding_head.3.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm2.weight 256 0 + backbone.0.q_norm.weight 16 0 + backbone.0.k_norm.weight 16 0 + backbone.0.ff.gate_proj.weight 262144 0 + backbone.0.ff.up_proj.weight 262144 0 + backbone.0.ff.down_proj.weight 262144 0 + backbone.1.regions 0 96 + backbone.1.attn.out_linear.weight 65536 0 + backbone.1.attn.out_linear.bias 256 0 + backbone.1.attn.e2lsh.alpha 0 864 + backbone.1.attn.e2lsh.beta 0 3 + backbone.1.attn.e2lsh_new.alpha 0 96 + backbone.1.attn.e2lsh_new.beta 0 3 + backbone.1.w_q.weight 65536 0 + backbone.1.w_k.weight 65536 0 + backbone.1.w_v.weight 65536 0 + backbone.1.w_rpe.weight 0 2560 + backbone.1.w_rpe.bias 0 256 +backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + backbone.1.pe_func.position_embedding_head.0.bias 1024 0 +backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + backbone.1.pe_func.position_embedding_head.1.bias 1024 0 +backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + backbone.1.pe_func.position_embedding_head.3.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm2.weight 256 0 + backbone.1.q_norm.weight 16 0 + backbone.1.k_norm.weight 16 0 + backbone.1.ff.gate_proj.weight 262144 0 + backbone.1.ff.up_proj.weight 262144 0 + backbone.1.ff.down_proj.weight 262144 0 + backbone.2.regions 0 96 + backbone.2.attn.out_linear.weight 65536 0 + backbone.2.attn.out_linear.bias 256 0 + backbone.2.attn.e2lsh.alpha 0 864 + backbone.2.attn.e2lsh.beta 0 3 + backbone.2.attn.e2lsh_new.alpha 0 96 + backbone.2.attn.e2lsh_new.beta 0 3 + backbone.2.w_q.weight 65536 0 + backbone.2.w_k.weight 65536 0 + backbone.2.w_v.weight 65536 0 + backbone.2.w_rpe.weight 0 2560 + backbone.2.w_rpe.bias 0 256 +backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + backbone.2.pe_func.position_embedding_head.0.bias 1024 0 +backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + backbone.2.pe_func.position_embedding_head.1.bias 1024 0 +backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + backbone.2.pe_func.position_embedding_head.3.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm2.weight 256 0 + backbone.2.q_norm.weight 16 0 + backbone.2.k_norm.weight 16 0 + backbone.2.ff.gate_proj.weight 262144 0 + backbone.2.ff.up_proj.weight 262144 0 + backbone.2.ff.down_proj.weight 262144 0 + backbone.3.regions 0 96 + backbone.3.attn.out_linear.weight 65536 0 + backbone.3.attn.out_linear.bias 256 0 + backbone.3.attn.e2lsh.alpha 0 864 + backbone.3.attn.e2lsh.beta 0 3 + backbone.3.attn.e2lsh_new.alpha 0 96 + backbone.3.attn.e2lsh_new.beta 0 3 + backbone.3.w_q.weight 65536 0 + backbone.3.w_k.weight 65536 0 + backbone.3.w_v.weight 65536 0 + backbone.3.w_rpe.weight 0 2560 + backbone.3.w_rpe.bias 0 256 +backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + backbone.3.pe_func.position_embedding_head.0.bias 1024 0 +backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + backbone.3.pe_func.position_embedding_head.1.bias 1024 0 +backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + backbone.3.pe_func.position_embedding_head.3.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm2.weight 256 0 + backbone.3.q_norm.weight 16 0 + backbone.3.k_norm.weight 16 0 + backbone.3.ff.gate_proj.weight 262144 0 + backbone.3.ff.up_proj.weight 262144 0 + backbone.3.ff.down_proj.weight 262144 0 + backbone.4.regions 0 96 + backbone.4.attn.out_linear.weight 65536 0 + backbone.4.attn.out_linear.bias 256 0 + backbone.4.attn.e2lsh.alpha 0 864 + backbone.4.attn.e2lsh.beta 0 3 + backbone.4.attn.e2lsh_new.alpha 0 96 + backbone.4.attn.e2lsh_new.beta 0 3 + backbone.4.w_q.weight 65536 0 + backbone.4.w_k.weight 65536 0 + backbone.4.w_v.weight 65536 0 + backbone.4.w_rpe.weight 0 2560 + backbone.4.w_rpe.bias 0 256 +backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + backbone.4.pe_func.position_embedding_head.0.bias 1024 0 +backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + backbone.4.pe_func.position_embedding_head.1.bias 1024 0 +backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + backbone.4.pe_func.position_embedding_head.3.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm2.weight 256 0 + backbone.4.q_norm.weight 16 0 + backbone.4.k_norm.weight 16 0 + backbone.4.ff.gate_proj.weight 262144 0 + backbone.4.ff.up_proj.weight 262144 0 + backbone.4.ff.down_proj.weight 262144 0 + backbone.5.regions 0 96 + backbone.5.attn.out_linear.weight 65536 0 + backbone.5.attn.out_linear.bias 256 0 + backbone.5.attn.e2lsh.alpha 0 864 + backbone.5.attn.e2lsh.beta 0 3 + backbone.5.attn.e2lsh_new.alpha 0 96 + backbone.5.attn.e2lsh_new.beta 0 3 + backbone.5.w_q.weight 65536 0 + backbone.5.w_k.weight 65536 0 + backbone.5.w_v.weight 65536 0 + backbone.5.w_rpe.weight 0 2560 + backbone.5.w_rpe.bias 0 256 +backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + backbone.5.pe_func.position_embedding_head.0.bias 1024 0 +backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + backbone.5.pe_func.position_embedding_head.1.bias 1024 0 +backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + backbone.5.pe_func.position_embedding_head.3.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm2.weight 256 0 + backbone.5.q_norm.weight 16 0 + backbone.5.k_norm.weight 16 0 + backbone.5.ff.gate_proj.weight 262144 0 + backbone.5.ff.up_proj.weight 262144 0 + backbone.5.ff.down_proj.weight 262144 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,364] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020 +rank0 [2026-10-28 11:12:25,364] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020 +rank0 [2026-10-28 11:12:25,364] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,364] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,276] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,312] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,343] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,375] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,402] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,442] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,464] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,490] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,514] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,542] INFO : train_dataset: cld_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,542] INFO : sample=cld_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,561] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,596] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,646] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,694] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,733] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,762] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,789] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,815] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,840] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,865] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,865] INFO : sample=cld_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,901] INFO : train_dataset: cld_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:32,923] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,966] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,993] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,016] INFO : train_dataset: cld_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,052] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,080] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,122] INFO : train_dataset: cld_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,148] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset: cld_edm_qq_pf, 90598 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,178] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,178] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,178] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,183] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,189] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,195] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,200] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,206] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,211] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,217] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,223] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,228] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,234] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,234] INFO : sample=cld_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,239] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,244] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,249] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,254] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,259] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,264] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,269] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,274] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,279] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,284] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,285] INFO : sample=cld_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,289] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,294] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,299] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,304] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,309] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,314] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,319] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,324] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,329] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,334] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,334] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,334] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,361] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,365] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,365] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-28 11:12:33,365] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,365] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,371] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,389] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,641] INFO : Created train_iterator +rank0 [2026-10-28 11:13:44,705] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.158649005889893, 'Classification': 0.11118821758776903, 'Regression_pt': 0.43017278850078583, 'Regression_eta': 0.10493254894390702, 'Regression_sin_phi': 0.08165183812379836, 'Regression_cos_phi': 0.07327452296391129, 'Regression_energy': 0.4708075523376465}, weights={'Classification_binary': 1.0, 'Classification': 46.39564514160156, 'Regression_pt': 11.992039680480957, 'Regression_eta': 49.16157150268555, 'Regression_sin_phi': 63.17860412597656, 'Regression_cos_phi': 70.40167236328125, 'Regression_energy': 10.957022666931152} +rank0 [2026-10-28 11:13:45,891] INFO : Step 100/50000 rank0 | Train Loss: 5.4789 | Train Loss Unweighted: 5.4789 | Train Loss Components: Classification: 0.0486 | Classification_binary: 4.6932 | Regression_cos_phi: 0.0244 | Regression_energy: 0.3346 | Regression_eta: 0.0324 | Regression_pt: 0.3264 | Regression_sin_phi: 0.0193 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 4.6932 | Regression_cos_phi: 0.0244 | Regression_energy: 0.3346 | Regression_eta: 0.0324 | Regression_pt: 0.3264 | Regression_sin_phi: 0.0193 | LR: 1.00e-03 | DataLoad Time: 0.0018s | Model Forward Time: 1.6840s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.1291 | max_backward: 0.2024 | max_optimizer_step: 0.1173 | max_tensorboard_logging: 0.4304 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 2948.8 | uss_MB: 2142.2 | child_rss_MB: 18193.7 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:13:46,104] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:14:26,674] INFO : Step 200/50000 rank0 | Train Loss: 16.9002 | Train Loss Unweighted: 5.1575 | Train Loss Components: Classification: 0.0411 | Classification_binary: 4.4483 | Regression_cos_phi: 0.0126 | Regression_energy: 0.3124 | Regression_eta: 0.0350 | Regression_pt: 0.2921 | Regression_sin_phi: 0.0161 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.9087 | Classification_binary: 4.4483 | Regression_cos_phi: 0.8847 | Regression_energy: 3.4233 | Regression_eta: 1.7183 | Regression_pt: 3.5028 | Regression_sin_phi: 1.0142 | LR: 1.00e-03 | DataLoad Time: 0.0025s | Model Forward Time: 1.3585s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.3098 | max_optimizer_step: 0.1065 | max_tensorboard_logging: 0.2600 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 2948.5 | uss_MB: 2142.7 | child_rss_MB: 4127.1 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:14:26,888] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:07,444] INFO : Step 300/50000 rank0 | Train Loss: 15.8682 | Train Loss Unweighted: 5.2735 | Train Loss Components: Classification: 0.0381 | Classification_binary: 4.5983 | Regression_cos_phi: 0.0146 | Regression_energy: 0.2976 | Regression_eta: 0.0192 | Regression_pt: 0.2940 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7654 | Classification_binary: 4.5983 | Regression_cos_phi: 1.0277 | Regression_energy: 3.2610 | Regression_eta: 0.9441 | Regression_pt: 3.5251 | Regression_sin_phi: 0.7467 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.4438s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0915 | max_forward: 0.0770 | max_loss: 0.0030 | max_backward: 0.2021 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.2562 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2947.5 | uss_MB: 2142.8 | child_rss_MB: 4123.9 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:15:07,660] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:48,186] INFO : Step 400/50000 rank0 | Train Loss: 15.1895 | Train Loss Unweighted: 4.9275 | Train Loss Components: Classification: 0.0383 | Classification_binary: 4.2626 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2965 | Regression_eta: 0.0198 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7769 | Classification_binary: 4.2626 | Regression_cos_phi: 0.7469 | Regression_energy: 3.2491 | Regression_eta: 0.9717 | Regression_pt: 3.4568 | Regression_sin_phi: 0.7256 | LR: 1.00e-03 | DataLoad Time: 0.0019s | Model Forward Time: 1.4068s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1823 | max_loss: 0.0032 | max_backward: 0.2024 | max_optimizer_step: 0.1018 | max_tensorboard_logging: 0.2946 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 2947.0 | uss_MB: 2142.8 | child_rss_MB: 4121.4 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:15:48,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:30,174] INFO : Step 500/50000 rank0 | Train Loss: 15.0155 | Train Loss Unweighted: 4.9003 | Train Loss Components: Classification: 0.0386 | Classification_binary: 4.2703 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2761 | Regression_eta: 0.0220 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7927 | Classification_binary: 4.2703 | Regression_cos_phi: 0.7647 | Regression_energy: 3.0252 | Regression_eta: 1.0816 | Regression_pt: 3.2243 | Regression_sin_phi: 0.8568 | LR: 1.00e-03 | DataLoad Time: 0.0024s | Model Forward Time: 1.4569s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.1043 | max_loss: 0.0031 | max_backward: 0.3916 | max_optimizer_step: 0.1023 | max_tensorboard_logging: 0.2573 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2948.1 | uss_MB: 2144.9 | child_rss_MB: 4141.6 | allocated_MB: 295.1 | reserved_MB: 3800.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 11:16:30,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:11,141] INFO : Step 600/50000 rank0 | Train Loss: 15.6284 | Train Loss Unweighted: 4.8147 | Train Loss Components: Classification: 0.0357 | Classification_binary: 4.1430 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2976 | Regression_eta: 0.0373 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.6557 | Classification_binary: 4.1430 | Regression_cos_phi: 0.8000 | Regression_energy: 3.2613 | Regression_eta: 1.8338 | Regression_pt: 3.3660 | Regression_sin_phi: 0.5684 | LR: 1.00e-03 | DataLoad Time: 0.0023s | Model Forward Time: 1.3692s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3091 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2656 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 2883.6 | uss_MB: 2080.9 | child_rss_MB: 4138.6 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:17:11,352] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:51,965] INFO : Step 700/50000 rank0 | Train Loss: 14.7855 | Train Loss Unweighted: 4.9106 | Train Loss Components: Classification: 0.0371 | Classification_binary: 4.2729 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2808 | Regression_eta: 0.0175 | Regression_pt: 0.2797 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7223 | Classification_binary: 4.2729 | Regression_cos_phi: 0.7401 | Regression_energy: 3.0773 | Regression_eta: 0.8611 | Regression_pt: 3.3545 | Regression_sin_phi: 0.7574 | LR: 1.00e-03 | DataLoad Time: 0.0031s | Model Forward Time: 1.5770s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0963 | max_forward: 0.1040 | max_loss: 0.0032 | max_backward: 0.3922 | max_optimizer_step: 0.1029 | max_tensorboard_logging: 0.2810 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2886.5 | uss_MB: 2085.1 | child_rss_MB: 4115.4 | allocated_MB: 295.1 | reserved_MB: 3800.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:17:52,177] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:32,737] INFO : Step 800/50000 rank0 | Train Loss: 14.3978 | Train Loss Unweighted: 4.7712 | Train Loss Components: Classification: 0.0343 | Classification_binary: 4.1502 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2764 | Regression_eta: 0.0180 | Regression_pt: 0.2696 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5931 | Classification_binary: 4.1502 | Regression_cos_phi: 0.7676 | Regression_energy: 3.0284 | Regression_eta: 0.8860 | Regression_pt: 3.2333 | Regression_sin_phi: 0.7392 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2614s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.2022 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.2494 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2884.9 | uss_MB: 2083.5 | child_rss_MB: 4115.0 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:18:32,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:13,599] INFO : Step 900/50000 rank0 | Train Loss: 13.1860 | Train Loss Unweighted: 4.7561 | Train Loss Components: Classification: 0.0338 | Classification_binary: 4.2109 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2406 | Regression_eta: 0.0145 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5695 | Classification_binary: 4.2109 | Regression_cos_phi: 0.6346 | Regression_energy: 2.6367 | Regression_eta: 0.7112 | Regression_pt: 2.8582 | Regression_sin_phi: 0.5650 | LR: 9.99e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0711 | max_backward: 0.2024 | max_optimizer_step: 0.1010 | max_tensorboard_logging: 0.4631 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2884.9 | uss_MB: 2083.5 | child_rss_MB: 4115.6 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:19:13,812] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:54,516] INFO : Step 1000/50000 rank0 | Train Loss: 14.1952 | Train Loss Unweighted: 4.7871 | Train Loss Components: Classification: 0.0387 | Classification_binary: 4.2168 | Regression_cos_phi: 0.0121 | Regression_energy: 0.2518 | Regression_eta: 0.0195 | Regression_pt: 0.2359 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7974 | Classification_binary: 4.2168 | Regression_cos_phi: 0.8521 | Regression_energy: 2.7586 | Regression_eta: 0.9598 | Regression_pt: 2.8287 | Regression_sin_phi: 0.7818 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3137s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1218 | max_loss: 0.0029 | max_backward: 0.2028 | max_optimizer_step: 0.1013 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2885.0 | uss_MB: 2083.6 | child_rss_MB: 4115.7 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:19:54,734] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:19:54,807] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:19:54,983] INFO : Running validation on rank0 +rank0 [2026-10-28 11:19:54,983] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:19:54,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:19:57,514] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:19:57,701] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:19:57,803] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:19:57,865] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:19:57,931] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:19:57,982] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:19:58,043] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:19:58,105] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:19:58,159] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:19:58,254] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:19:58,353] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:19:58,413] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:19:58,475] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:19:58,526] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:19:58,589] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:19:58,653] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:19:58,711] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:19:58,715] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:19:58,718] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:19:58,720] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:19:58,721] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:19:58,724] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:19:58,727] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:19:58,728] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:19:58,731] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:19:58,734] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:19:58,736] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:19:58,737] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:19:58,740] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:19:58,743] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:19:58,745] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:19:58,746] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:20:06,293] INFO : VALIDATION | Step=1000/50000 | Train Loss=14.1952 | Valid Loss=13.7567 | Train Loss Unweighted=4.7871 | Valid Loss Unweighted=4.7167 | Stale=0 | ETA=369.9m +rank0 [2026-10-28 11:20:06,300] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:06,361] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:20:06,371] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:06,506] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:08,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:09,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:10,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:10,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:10,817] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:20:10,820] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:10,875] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:10,878] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:10,963] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:12,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:13,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:14,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:14,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:14,897] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:20:14,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:14,954] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:14,957] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:20:15,099] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:16,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:17,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:18,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:18,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:18,986] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:18,994] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:19,253] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:19,802] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:21,656] INFO : Plotted jets +rank0 [2026-10-28 11:20:22,730] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:22,765] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:22,996] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:23,384] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:24,769] INFO : Plotted jets +rank0 [2026-10-28 11:20:25,870] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:25,904] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:26,110] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:26,476] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:27,839] INFO : Plotted jets +rank0 [2026-10-28 11:20:28,874] INFO : Plotted jet ratio +rank0 [2026-10-28 11:21:09,928] INFO : Step 1100/50000 rank0 | Train Loss: 14.0203 | Train Loss Unweighted: 4.5956 | Train Loss Components: Classification: 0.0355 | Classification_binary: 4.0186 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2509 | Regression_eta: 0.0268 | Regression_pt: 0.2432 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.6471 | Classification_binary: 4.0186 | Regression_cos_phi: 0.7384 | Regression_energy: 2.7488 | Regression_eta: 1.3165 | Regression_pt: 2.9169 | Regression_sin_phi: 0.6339 | LR: 9.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4666s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0768 | max_loss: 0.0145 | max_backward: 0.2024 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3342 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3184.9 | uss_MB: 2392.5 | child_rss_MB: 17755.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:21:10,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:51,081] INFO : Step 1200/50000 rank0 | Train Loss: 13.5316 | Train Loss Unweighted: 4.5611 | Train Loss Components: Classification: 0.0342 | Classification_binary: 4.0019 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2466 | Regression_eta: 0.0192 | Regression_pt: 0.2374 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5889 | Classification_binary: 4.0019 | Regression_cos_phi: 0.7376 | Regression_energy: 2.7016 | Regression_eta: 0.9430 | Regression_pt: 2.8470 | Regression_sin_phi: 0.7115 | LR: 9.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4305s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0522 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.2641 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3184.2 | uss_MB: 2392.9 | child_rss_MB: 4111.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:21:51,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:33,381] INFO : Step 1300/50000 rank0 | Train Loss: 14.4450 | Train Loss Unweighted: 4.6261 | Train Loss Components: Classification: 0.0328 | Classification_binary: 4.0426 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2520 | Regression_eta: 0.0326 | Regression_pt: 0.2420 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5206 | Classification_binary: 4.0426 | Regression_cos_phi: 0.8621 | Regression_energy: 2.7613 | Regression_eta: 1.6017 | Regression_pt: 2.9023 | Regression_sin_phi: 0.7545 | LR: 9.98e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6043s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1758 | max_loss: 0.0030 | max_backward: 0.2868 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 0.3130 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3185.8 | uss_MB: 2394.8 | child_rss_MB: 4107.7 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:22:33,613] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:14,337] INFO : Step 1400/50000 rank0 | Train Loss: 13.5429 | Train Loss Unweighted: 4.5340 | Train Loss Components: Classification: 0.0318 | Classification_binary: 3.9808 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2429 | Regression_eta: 0.0247 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0127 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4767 | Classification_binary: 3.9808 | Regression_cos_phi: 0.6242 | Regression_energy: 2.6612 | Regression_eta: 1.2133 | Regression_pt: 2.7849 | Regression_sin_phi: 0.8019 | LR: 9.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4952s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0720 | max_forward: 0.0774 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.2882 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3182.6 | uss_MB: 2391.4 | child_rss_MB: 4109.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:23:14,566] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:55,775] INFO : Step 1500/50000 rank0 | Train Loss: 13.0110 | Train Loss Unweighted: 4.5981 | Train Loss Components: Classification: 0.0317 | Classification_binary: 4.0673 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2360 | Regression_eta: 0.0184 | Regression_pt: 0.2255 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4689 | Classification_binary: 4.0673 | Regression_cos_phi: 0.6535 | Regression_energy: 2.5859 | Regression_eta: 0.9058 | Regression_pt: 2.7043 | Regression_sin_phi: 0.6252 | LR: 9.98e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4262s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1843 | max_loss: 0.0031 | max_backward: 0.2027 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.2826 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3182.5 | uss_MB: 2391.4 | child_rss_MB: 4106.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:23:55,986] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:36,718] INFO : Step 1600/50000 rank0 | Train Loss: 13.4222 | Train Loss Unweighted: 4.5041 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.9498 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2433 | Regression_eta: 0.0220 | Regression_pt: 0.2359 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4711 | Classification_binary: 3.9498 | Regression_cos_phi: 0.7589 | Regression_energy: 2.6664 | Regression_eta: 1.0816 | Regression_pt: 2.8294 | Regression_sin_phi: 0.6649 | LR: 9.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4503s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1839 | max_loss: 0.0030 | max_backward: 0.2025 | max_optimizer_step: 0.1073 | max_tensorboard_logging: 0.2999 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3043.7 | uss_MB: 2253.1 | child_rss_MB: 4105.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:24:36,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:17,499] INFO : Step 1700/50000 rank0 | Train Loss: 12.2893 | Train Loss Unweighted: 4.4091 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.9110 | Regression_cos_phi: 0.0076 | Regression_energy: 0.2217 | Regression_eta: 0.0190 | Regression_pt: 0.2102 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5035 | Classification_binary: 3.9110 | Regression_cos_phi: 0.5340 | Regression_energy: 2.4292 | Regression_eta: 0.9321 | Regression_pt: 2.5210 | Regression_sin_phi: 0.4585 | LR: 9.97e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4236s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0771 | max_loss: 0.0028 | max_backward: 0.3109 | max_optimizer_step: 0.1055 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3044.7 | uss_MB: 2255.2 | child_rss_MB: 4107.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:25:17,713] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:57,330] INFO : Step 1800/50000 rank0 | Train Loss: 12.9007 | Train Loss Unweighted: 4.5419 | Train Loss Components: Classification: 0.0326 | Classification_binary: 4.0293 | Regression_cos_phi: 0.0099 | Regression_energy: 0.2226 | Regression_eta: 0.0203 | Regression_pt: 0.2174 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5146 | Classification_binary: 4.0293 | Regression_cos_phi: 0.6991 | Regression_energy: 2.4392 | Regression_eta: 0.9992 | Regression_pt: 2.6066 | Regression_sin_phi: 0.6127 | LR: 9.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5004s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.3076 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.3799 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3045.3 | uss_MB: 2255.3 | child_rss_MB: 4101.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:25:57,544] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:38,300] INFO : Step 1900/50000 rank0 | Train Loss: 12.1641 | Train Loss Unweighted: 4.3945 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.8891 | Regression_cos_phi: 0.0086 | Regression_energy: 0.2242 | Regression_eta: 0.0123 | Regression_pt: 0.2214 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3955 | Classification_binary: 3.8891 | Regression_cos_phi: 0.6042 | Regression_energy: 2.4568 | Regression_eta: 0.6056 | Regression_pt: 2.6549 | Regression_sin_phi: 0.5580 | LR: 9.97e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.6177s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1042 | max_loss: 0.0031 | max_backward: 0.3865 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3047.1 | uss_MB: 2257.8 | child_rss_MB: 4098.4 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:26:38,509] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:19,021] INFO : Step 2000/50000 rank0 | Train Loss: 13.0496 | Train Loss Unweighted: 4.4408 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.8727 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2549 | Regression_eta: 0.0148 | Regression_pt: 0.2484 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5202 | Classification_binary: 3.8727 | Regression_cos_phi: 0.6327 | Regression_energy: 2.7932 | Regression_eta: 0.7266 | Regression_pt: 2.9786 | Regression_sin_phi: 0.5256 | LR: 9.96e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4105s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3107 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3044.9 | uss_MB: 2256.6 | child_rss_MB: 4101.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:27:19,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:19,252] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:27:19,425] INFO : Running validation on rank0 +rank0 [2026-10-28 11:27:19,425] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:27:19,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:20,741] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:27:20,885] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:21,016] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:21,091] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:21,161] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:21,229] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:21,298] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:21,367] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:21,469] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:21,600] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:21,731] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:21,801] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:21,869] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:21,938] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:22,007] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:22,076] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:22,177] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:22,184] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:22,187] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:22,190] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:22,193] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:22,195] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:22,197] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:22,199] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:22,203] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:22,206] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:22,207] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:22,209] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:22,212] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:22,213] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:22,216] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:22,218] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:29,914] INFO : VALIDATION | Step=2000/50000 | Train Loss=13.0496 | Valid Loss=12.6006 | Train Loss Unweighted=4.4408 | Valid Loss Unweighted=4.4343 | Stale=0 | ETA=358.6m +rank0 [2026-10-28 11:27:29,915] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:29,973] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:29,977] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:30,115] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:32,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:32,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:33,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:34,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:34,250] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:27:34,253] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:34,309] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:34,312] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:34,390] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:36,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:36,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:37,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:38,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:38,344] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:27:38,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:38,401] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:38,404] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:27:38,556] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:40,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:41,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:41,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:42,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:42,437] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:42,445] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:42,700] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:43,102] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:44,943] INFO : Plotted jets +rank0 [2026-10-28 11:27:46,024] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:46,056] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:46,283] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:46,676] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:48,070] INFO : Plotted jets +rank0 [2026-10-28 11:27:49,174] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:49,208] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:49,408] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:49,771] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:51,144] INFO : Plotted jets +rank0 [2026-10-28 11:27:52,187] INFO : Plotted jet ratio +rank0 [2026-10-28 11:28:33,384] INFO : Step 2100/50000 rank0 | Train Loss: 13.3832 | Train Loss Unweighted: 4.4776 | Train Loss Components: Classification: 0.0325 | Classification_binary: 3.9215 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2467 | Regression_eta: 0.0188 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5079 | Classification_binary: 3.9215 | Regression_cos_phi: 0.7744 | Regression_energy: 2.7032 | Regression_eta: 0.9249 | Regression_pt: 2.8247 | Regression_sin_phi: 0.7266 | LR: 9.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1856 | max_loss: 0.0030 | max_backward: 0.2029 | max_optimizer_step: 0.1100 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3281.3 | uss_MB: 2495.3 | child_rss_MB: 4047.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:28:33,597] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:14,317] INFO : Step 2200/50000 rank0 | Train Loss: 11.9067 | Train Loss Unweighted: 4.3015 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.8337 | Regression_cos_phi: 0.0097 | Regression_energy: 0.2070 | Regression_eta: 0.0160 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5029 | Classification_binary: 3.8337 | Regression_cos_phi: 0.6858 | Regression_energy: 2.2677 | Regression_eta: 0.7876 | Regression_pt: 2.3368 | Regression_sin_phi: 0.4921 | LR: 9.96e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1038 | max_loss: 0.0031 | max_backward: 0.3373 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3858 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3282.9 | uss_MB: 2497.4 | child_rss_MB: 4049.0 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:29:14,529] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:55,853] INFO : Step 2300/50000 rank0 | Train Loss: 12.2761 | Train Loss Unweighted: 4.4087 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.8831 | Regression_cos_phi: 0.0068 | Regression_energy: 0.2374 | Regression_eta: 0.0154 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3783 | Classification_binary: 3.8831 | Regression_cos_phi: 0.4780 | Regression_energy: 2.6016 | Regression_eta: 0.7567 | Regression_pt: 2.7521 | Regression_sin_phi: 0.4263 | LR: 9.95e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4669s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1822 | max_loss: 0.0030 | max_backward: 0.2023 | max_optimizer_step: 0.1074 | max_tensorboard_logging: 0.3016 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3282.3 | uss_MB: 2495.4 | child_rss_MB: 4042.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:29:56,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:37,466] INFO : Step 2400/50000 rank0 | Train Loss: 12.4005 | Train Loss Unweighted: 4.3180 | Train Loss Components: Classification: 0.0290 | Classification_binary: 3.8035 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2286 | Regression_eta: 0.0145 | Regression_pt: 0.2217 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3449 | Classification_binary: 3.8035 | Regression_cos_phi: 0.6727 | Regression_energy: 2.5044 | Regression_eta: 0.7148 | Regression_pt: 2.6588 | Regression_sin_phi: 0.7015 | LR: 9.95e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4109s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1844 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1074 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3282.3 | uss_MB: 2495.4 | child_rss_MB: 4043.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:30:37,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:18,295] INFO : Step 2500/50000 rank0 | Train Loss: 11.5982 | Train Loss Unweighted: 4.1917 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.7021 | Regression_cos_phi: 0.0083 | Regression_energy: 0.2178 | Regression_eta: 0.0105 | Regression_pt: 0.2166 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3275 | Classification_binary: 3.7021 | Regression_cos_phi: 0.5857 | Regression_energy: 2.3867 | Regression_eta: 0.5172 | Regression_pt: 2.5980 | Regression_sin_phi: 0.4809 | LR: 9.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4622s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0767 | max_loss: 0.0029 | max_backward: 0.2284 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3822 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3282.3 | uss_MB: 2495.4 | child_rss_MB: 4042.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:31:18,507] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:00,037] INFO : Step 2600/50000 rank0 | Train Loss: 11.0203 | Train Loss Unweighted: 4.1058 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.6802 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1851 | Regression_eta: 0.0110 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3677 | Classification_binary: 3.6802 | Regression_cos_phi: 0.5588 | Regression_energy: 2.0280 | Regression_eta: 0.5413 | Regression_pt: 2.1767 | Regression_sin_phi: 0.6676 | LR: 9.94e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6175s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1041 | max_loss: 0.0030 | max_backward: 0.3429 | max_optimizer_step: 0.1071 | max_tensorboard_logging: 0.4077 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.2 | uss_MB: 2332.4 | child_rss_MB: 4042.4 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:32:00,252] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:40,998] INFO : Step 2700/50000 rank0 | Train Loss: 12.4271 | Train Loss Unweighted: 4.3367 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.7987 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2402 | Regression_eta: 0.0114 | Regression_pt: 0.2391 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3551 | Classification_binary: 3.7987 | Regression_cos_phi: 0.6738 | Regression_energy: 2.6319 | Regression_eta: 0.5583 | Regression_pt: 2.8672 | Regression_sin_phi: 0.5422 | LR: 9.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3870s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1323 | max_loss: 0.0029 | max_backward: 0.2032 | max_optimizer_step: 0.1076 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.2 | uss_MB: 2331.8 | child_rss_MB: 4043.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:32:41,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:21,453] INFO : Step 2800/50000 rank0 | Train Loss: 11.5494 | Train Loss Unweighted: 4.2442 | Train Loss Components: Classification: 0.0290 | Classification_binary: 3.8007 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1928 | Regression_eta: 0.0170 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3465 | Classification_binary: 3.8007 | Regression_cos_phi: 0.6187 | Regression_energy: 2.1125 | Regression_eta: 0.8348 | Regression_pt: 2.2366 | Regression_sin_phi: 0.5995 | LR: 9.93e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4697s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0543 | max_forward: 0.0769 | max_loss: 0.0031 | max_backward: 0.2019 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.2848 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3118.2 | uss_MB: 2332.4 | child_rss_MB: 4039.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:33:21,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:01,729] INFO : Step 2900/50000 rank0 | Train Loss: 10.9168 | Train Loss Unweighted: 4.1168 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.6592 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2046 | Regression_eta: 0.0090 | Regression_pt: 0.2035 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2934 | Classification_binary: 3.6592 | Regression_cos_phi: 0.4482 | Regression_energy: 2.2422 | Regression_eta: 0.4441 | Regression_pt: 2.4407 | Regression_sin_phi: 0.3891 | LR: 9.93e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5006s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0842 | max_forward: 0.0776 | max_loss: 0.0029 | max_backward: 0.2025 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.2788 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.2 | uss_MB: 2332.6 | child_rss_MB: 4039.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:34:01,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:42,395] INFO : Step 3000/50000 rank0 | Train Loss: 11.9253 | Train Loss Unweighted: 4.1456 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.6636 | Regression_cos_phi: 0.0118 | Regression_energy: 0.2114 | Regression_eta: 0.0152 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3161 | Classification_binary: 3.6636 | Regression_cos_phi: 0.8298 | Regression_energy: 2.3165 | Regression_eta: 0.7459 | Regression_pt: 2.4707 | Regression_sin_phi: 0.5826 | LR: 9.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5329s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1034 | max_loss: 0.0556 | max_backward: 0.2863 | max_optimizer_step: 0.1064 | max_tensorboard_logging: 0.3339 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.7 | uss_MB: 2335.5 | child_rss_MB: 4039.5 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:34:42,609] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:42,660] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:34:42,835] INFO : Running validation on rank0 +rank0 [2026-10-28 11:34:42,835] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:34:42,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:34:44,155] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:34:44,299] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:34:44,430] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:34:44,504] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:34:44,574] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:34:44,642] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:34:44,712] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:34:44,780] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:34:44,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:34:45,012] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:34:45,143] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:34:45,214] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:34:45,282] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:34:45,350] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:34:45,418] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:34:45,487] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:34:45,604] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:34:45,612] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:34:45,614] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:34:45,617] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:34:45,618] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:34:45,621] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:34:45,623] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:34:45,625] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:34:45,628] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:34:45,631] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:34:45,633] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:34:45,636] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:34:45,637] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:34:45,640] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:34:45,642] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:34:45,644] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:34:53,157] INFO : VALIDATION | Step=3000/50000 | Train Loss=11.9253 | Valid Loss=11.6295 | Train Loss Unweighted=4.1456 | Valid Loss Unweighted=4.1618 | Stale=0 | ETA=349.8m +rank0 [2026-10-28 11:34:53,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:53,230] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:34:53,233] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:53,377] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:55,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:55,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:56,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:57,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:57,385] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:34:57,388] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:57,443] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:34:57,446] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:57,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:59,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:35:00,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:35:00,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:35:01,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:35:01,486] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:35:01,490] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:35:01,544] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:35:01,547] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:35:01,633] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:35:03,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:35:04,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:35:04,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:35:05,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:35:05,561] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:35:05,569] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:35:05,826] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:35:06,229] INFO : Plotted number of elements +rank0 [2026-10-28 11:35:08,090] INFO : Plotted jets +rank0 [2026-10-28 11:35:09,170] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:09,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:35:09,431] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:35:09,819] INFO : Plotted number of elements +rank0 [2026-10-28 11:35:11,210] INFO : Plotted jets +rank0 [2026-10-28 11:35:12,318] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:12,351] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:35:12,554] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:35:12,915] INFO : Plotted number of elements +rank0 [2026-10-28 11:35:14,287] INFO : Plotted jets +rank0 [2026-10-28 11:35:15,718] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:57,289] INFO : Step 3100/50000 rank0 | Train Loss: 12.0174 | Train Loss Unweighted: 4.2579 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.7798 | Regression_cos_phi: 0.0086 | Regression_energy: 0.2099 | Regression_eta: 0.0186 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3918 | Classification_binary: 3.7798 | Regression_cos_phi: 0.6035 | Regression_energy: 2.3003 | Regression_eta: 0.9140 | Regression_pt: 2.4145 | Regression_sin_phi: 0.6136 | LR: 9.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.3110 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.3877 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4037.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:35:57,500] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:38,300] INFO : Step 3200/50000 rank0 | Train Loss: 11.1021 | Train Loss Unweighted: 4.0789 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.6534 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1822 | Regression_eta: 0.0148 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4012 | Classification_binary: 3.6534 | Regression_cos_phi: 0.6408 | Regression_energy: 1.9959 | Regression_eta: 0.7300 | Regression_pt: 2.1717 | Regression_sin_phi: 0.5092 | LR: 9.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4908s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0626 | max_backward: 0.2029 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4038.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 11:36:38,511] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:19,121] INFO : Step 3300/50000 rank0 | Train Loss: 12.4079 | Train Loss Unweighted: 4.2953 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.7670 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2343 | Regression_eta: 0.0141 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3848 | Classification_binary: 3.7670 | Regression_cos_phi: 0.6463 | Regression_energy: 2.5669 | Regression_eta: 0.6915 | Regression_pt: 2.7804 | Regression_sin_phi: 0.5709 | LR: 9.90e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0767 | max_loss: 0.0030 | max_backward: 0.2743 | max_optimizer_step: 0.1086 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4034.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:37:19,332] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:00,624] INFO : Step 3400/50000 rank0 | Train Loss: 12.2776 | Train Loss Unweighted: 4.2387 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.7132 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2370 | Regression_eta: 0.0168 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2652 | Classification_binary: 3.7132 | Regression_cos_phi: 0.6471 | Regression_energy: 2.5969 | Regression_eta: 0.8245 | Regression_pt: 2.7242 | Regression_sin_phi: 0.5066 | LR: 9.90e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5363s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.3107 | max_optimizer_step: 0.1078 | max_tensorboard_logging: 0.3765 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4035.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:38:00,836] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:41,625] INFO : Step 3500/50000 rank0 | Train Loss: 11.4570 | Train Loss Unweighted: 4.0172 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.5304 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2177 | Regression_eta: 0.0108 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3115 | Classification_binary: 3.5304 | Regression_cos_phi: 0.6490 | Regression_energy: 2.3857 | Regression_eta: 0.5291 | Regression_pt: 2.5541 | Regression_sin_phi: 0.4971 | LR: 9.89e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5616s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0032 | max_backward: 0.3069 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.3847 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3339.1 | uss_MB: 2556.9 | child_rss_MB: 4035.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:38:41,837] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:21,940] INFO : Step 3600/50000 rank0 | Train Loss: 11.1187 | Train Loss Unweighted: 3.9992 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1781 | Regression_eta: 0.0213 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3090 | Classification_binary: 3.5823 | Regression_cos_phi: 0.6056 | Regression_energy: 1.9519 | Regression_eta: 1.0490 | Regression_pt: 2.0596 | Regression_sin_phi: 0.5613 | LR: 9.89e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0030 | max_backward: 0.2324 | max_optimizer_step: 0.1094 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3190.3 | uss_MB: 2406.6 | child_rss_MB: 4034.0 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:39:22,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:03,621] INFO : Step 3700/50000 rank0 | Train Loss: 12.8052 | Train Loss Unweighted: 3.9794 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.4605 | Regression_cos_phi: 0.0095 | Regression_energy: 0.2274 | Regression_eta: 0.0350 | Regression_pt: 0.2083 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3185 | Classification_binary: 3.4605 | Regression_cos_phi: 0.6658 | Regression_energy: 2.4914 | Regression_eta: 1.7199 | Regression_pt: 2.4980 | Regression_sin_phi: 0.6513 | LR: 9.88e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4793s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0371 | max_backward: 0.2031 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3884 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3192.3 | uss_MB: 2408.2 | child_rss_MB: 4027.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:40:03,834] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:44,689] INFO : Step 3800/50000 rank0 | Train Loss: 10.8068 | Train Loss Unweighted: 4.0242 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.5899 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1910 | Regression_eta: 0.0116 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2185 | Classification_binary: 3.5899 | Regression_cos_phi: 0.5888 | Regression_energy: 2.0931 | Regression_eta: 0.5686 | Regression_pt: 2.2732 | Regression_sin_phi: 0.4747 | LR: 9.87e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4594s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0501 | max_forward: 0.0770 | max_loss: 0.0037 | max_backward: 0.2025 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3192.3 | uss_MB: 2408.9 | child_rss_MB: 4030.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:40:44,913] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:26,709] INFO : Step 3900/50000 rank0 | Train Loss: 11.5790 | Train Loss Unweighted: 3.9766 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.5032 | Regression_cos_phi: 0.0080 | Regression_energy: 0.2088 | Regression_eta: 0.0200 | Regression_pt: 0.2002 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2782 | Classification_binary: 3.5032 | Regression_cos_phi: 0.5600 | Regression_energy: 2.2875 | Regression_eta: 0.9818 | Regression_pt: 2.4009 | Regression_sin_phi: 0.5673 | LR: 9.87e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4467s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1835 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.3085 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3192.3 | uss_MB: 2409.3 | child_rss_MB: 4030.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:41:26,920] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:07,850] INFO : Step 4000/50000 rank0 | Train Loss: 10.8825 | Train Loss Unweighted: 3.9858 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.5208 | Regression_cos_phi: 0.0070 | Regression_energy: 0.2084 | Regression_eta: 0.0090 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1961 | Classification_binary: 3.5208 | Regression_cos_phi: 0.4957 | Regression_energy: 2.2830 | Regression_eta: 0.4404 | Regression_pt: 2.4894 | Regression_sin_phi: 0.4572 | LR: 9.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4754s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0681 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2805 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3192.3 | uss_MB: 2409.3 | child_rss_MB: 4031.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:42:08,080] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:08,100] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:42:08,275] INFO : Running validation on rank0 +rank0 [2026-10-28 11:42:08,275] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:42:08,275] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:42:09,594] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:42:09,737] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:42:09,868] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:42:09,942] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:42:10,010] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:42:10,081] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:42:10,151] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:42:10,219] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:42:10,323] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:42:10,452] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:42:10,583] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:42:10,654] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:42:10,723] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:42:10,791] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:42:10,859] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:42:10,927] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:42:11,046] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:42:11,053] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:42:11,057] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:42:11,059] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:42:11,060] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:42:11,063] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:42:11,066] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:42:11,069] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:42:11,069] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:42:11,072] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:42:11,075] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:42:11,077] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:42:11,079] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:42:11,080] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:42:11,083] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:42:11,086] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:42:18,802] INFO : VALIDATION | Step=4000/50000 | Train Loss=10.8825 | Valid Loss=11.0341 | Train Loss Unweighted=3.9858 | Valid Loss Unweighted=3.9445 | Stale=0 | ETA=342.2m +rank0 [2026-10-28 11:42:18,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:42:18,877] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:42:18,881] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:42:18,969] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:42:20,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:42:21,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:42:22,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:42:22,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:42:22,727] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:42:22,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:42:22,785] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:42:22,788] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:42:22,864] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:42:24,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:42:25,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:42:25,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:42:26,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:42:26,646] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:42:26,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:42:26,704] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:42:26,707] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:42:26,856] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:42:28,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:42:29,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:42:29,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:42:30,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:42:30,762] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:42:30,769] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:42:31,028] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:42:31,434] INFO : Plotted number of elements +rank0 [2026-10-28 11:42:33,337] INFO : Plotted jets +rank0 [2026-10-28 11:42:34,418] INFO : Plotted jet ratio +rank0 [2026-10-28 11:42:34,449] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:42:34,676] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:42:35,062] INFO : Plotted number of elements +rank0 [2026-10-28 11:42:36,450] INFO : Plotted jets +rank0 [2026-10-28 11:42:37,548] INFO : Plotted jet ratio +rank0 [2026-10-28 11:42:37,580] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:42:37,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:42:38,148] INFO : Plotted number of elements +rank0 [2026-10-28 11:42:39,509] INFO : Plotted jets +rank0 [2026-10-28 11:42:40,982] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:22,124] INFO : Step 4100/50000 rank0 | Train Loss: 11.0773 | Train Loss Unweighted: 3.9121 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.4785 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1896 | Regression_eta: 0.0220 | Regression_pt: 0.1804 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1438 | Classification_binary: 3.4785 | Regression_cos_phi: 0.5597 | Regression_energy: 2.0770 | Regression_eta: 1.0808 | Regression_pt: 2.1628 | Regression_sin_phi: 0.5746 | LR: 9.85e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4603s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1841 | max_loss: 0.0033 | max_backward: 0.2021 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3397.6 | uss_MB: 2617.1 | child_rss_MB: 4026.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:43:22,341] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:02,522] INFO : Step 4200/50000 rank0 | Train Loss: 10.8108 | Train Loss Unweighted: 3.8229 | Train Loss Components: Classification: 0.0281 | Classification_binary: 3.4045 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1835 | Regression_eta: 0.0153 | Regression_pt: 0.1726 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3033 | Classification_binary: 3.4045 | Regression_cos_phi: 0.7144 | Regression_energy: 2.0111 | Regression_eta: 0.7508 | Regression_pt: 2.0695 | Regression_sin_phi: 0.5573 | LR: 9.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4730s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0272 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.2033 | max_optimizer_step: 0.1852 | max_tensorboard_logging: 0.2914 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4024.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 11:44:02,754] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:43,848] INFO : Step 4300/50000 rank0 | Train Loss: 10.6818 | Train Loss Unweighted: 3.7630 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.3376 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1858 | Regression_eta: 0.0190 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2174 | Classification_binary: 3.3376 | Regression_cos_phi: 0.5066 | Regression_energy: 2.0357 | Regression_eta: 0.9363 | Regression_pt: 2.1495 | Regression_sin_phi: 0.4988 | LR: 9.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4591s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1700 | max_loss: 0.0030 | max_backward: 0.2020 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3013 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4020.4 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 11:44:44,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:24,902] INFO : Step 4400/50000 rank0 | Train Loss: 10.2831 | Train Loss Unweighted: 3.8156 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.4102 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1775 | Regression_eta: 0.0104 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1185 | Classification_binary: 3.4102 | Regression_cos_phi: 0.6066 | Regression_energy: 1.9454 | Regression_eta: 0.5097 | Regression_pt: 2.1042 | Regression_sin_phi: 0.5884 | LR: 9.83e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6632s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0932 | max_forward: 0.1044 | max_loss: 0.0031 | max_backward: 0.3922 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0123 | Memory: rss_MB: 3397.6 | uss_MB: 2619.2 | child_rss_MB: 4047.5 | allocated_MB: 295.1 | reserved_MB: 3896.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 11:45:25,115] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:05,711] INFO : Step 4500/50000 rank0 | Train Loss: 10.7525 | Train Loss Unweighted: 3.6803 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.2673 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1796 | Regression_eta: 0.0166 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1778 | Classification_binary: 3.2673 | Regression_cos_phi: 0.8810 | Regression_energy: 1.9683 | Regression_eta: 0.8173 | Regression_pt: 2.0278 | Regression_sin_phi: 0.6130 | LR: 9.82e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4609s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0011 | max_forward: 0.0768 | max_loss: 0.0030 | max_backward: 0.3074 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.2948 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4048.0 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:46:05,924] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:46,758] INFO : Step 4600/50000 rank0 | Train Loss: 10.5031 | Train Loss Unweighted: 3.8296 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.4190 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1822 | Regression_eta: 0.0143 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2442 | Classification_binary: 3.4190 | Regression_cos_phi: 0.6101 | Regression_energy: 1.9964 | Regression_eta: 0.7033 | Regression_pt: 2.0508 | Regression_sin_phi: 0.4794 | LR: 9.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5928s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3087 | max_optimizer_step: 0.1124 | max_tensorboard_logging: 0.3965 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4025.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:46:46,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:27,461] INFO : Step 4700/50000 rank0 | Train Loss: 10.8839 | Train Loss Unweighted: 3.7758 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.3581 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1780 | Regression_eta: 0.0237 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2341 | Classification_binary: 3.3581 | Regression_cos_phi: 0.5954 | Regression_energy: 1.9507 | Regression_eta: 1.1656 | Regression_pt: 2.0727 | Regression_sin_phi: 0.5073 | LR: 9.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4757s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0769 | max_loss: 0.0030 | max_backward: 0.3092 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.3011 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3262.4 | uss_MB: 2481.6 | child_rss_MB: 4046.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 11:47:27,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:08,075] INFO : Step 4800/50000 rank0 | Train Loss: 11.8103 | Train Loss Unweighted: 3.9279 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.4434 | Regression_cos_phi: 0.0099 | Regression_energy: 0.2122 | Regression_eta: 0.0176 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3693 | Classification_binary: 3.4434 | Regression_cos_phi: 0.6991 | Regression_energy: 2.3245 | Regression_eta: 0.8653 | Regression_pt: 2.4586 | Regression_sin_phi: 0.6501 | LR: 9.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4230s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.1338 | max_loss: 0.0032 | max_backward: 0.2022 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.3073 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3262.4 | uss_MB: 2481.8 | child_rss_MB: 4019.8 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:48:08,290] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:48,903] INFO : Step 4900/50000 rank0 | Train Loss: 10.7401 | Train Loss Unweighted: 3.7777 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.3387 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1931 | Regression_eta: 0.0136 | Regression_pt: 0.1890 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2334 | Classification_binary: 3.3387 | Regression_cos_phi: 0.6055 | Regression_energy: 2.1160 | Regression_eta: 0.6675 | Regression_pt: 2.2663 | Regression_sin_phi: 0.5126 | LR: 9.79e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5204s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0769 | max_loss: 0.0857 | max_backward: 0.2024 | max_optimizer_step: 0.1063 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3262.4 | uss_MB: 2481.9 | child_rss_MB: 4018.9 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 11:48:49,116] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:29,750] INFO : Step 5000/50000 rank0 | Train Loss: 9.5813 | Train Loss Unweighted: 3.5707 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.1867 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1687 | Regression_eta: 0.0098 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1676 | Classification_binary: 3.1867 | Regression_cos_phi: 0.4537 | Regression_energy: 1.8484 | Regression_eta: 0.4822 | Regression_pt: 2.0014 | Regression_sin_phi: 0.4414 | LR: 9.78e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4425s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0031 | max_backward: 0.3081 | max_optimizer_step: 0.1060 | max_tensorboard_logging: 0.3052 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3262.4 | uss_MB: 2482.0 | child_rss_MB: 4046.3 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:49:29,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:29,984] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:49:30,156] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:30,156] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:30,156] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:49:31,377] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:49:31,522] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:49:31,654] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:49:31,728] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:49:31,797] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:49:31,865] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:49:31,935] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:49:32,004] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:49:32,107] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:49:32,238] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:49:32,368] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:49:32,439] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:49:32,507] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:49:32,576] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:49:32,645] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:49:32,714] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:49:32,817] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:49:32,824] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:49:32,826] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:49:32,829] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:49:32,832] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:49:32,833] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:49:32,836] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:49:32,839] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:49:32,840] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:49:32,843] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:49:32,845] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:49:32,846] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:49:32,849] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:49:32,850] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:49:32,853] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:49:32,856] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:49:40,462] INFO : VALIDATION | Step=5000/50000 | Train Loss=9.5813 | Valid Loss=10.5400 | Train Loss Unweighted=3.5707 | Valid Loss Unweighted=3.7617 | Stale=0 | ETA=334.1m +rank0 [2026-10-28 11:49:40,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:40,529] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:49:40,533] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:40,680] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:42,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:43,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:44,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:44,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:44,841] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:49:44,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:44,908] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:49:44,911] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:44,999] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:46,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:47,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:48,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:48,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:48,939] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:49:48,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:48,996] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:49:48,999] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:49:49,149] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:50,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:51,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:52,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:52,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:53,022] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:49:53,030] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:53,288] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:53,690] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:55,587] INFO : Plotted jets +rank0 [2026-10-28 11:49:56,668] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:56,702] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:56,932] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:57,320] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:58,713] INFO : Plotted jets +rank0 [2026-10-28 11:49:59,816] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:59,851] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:00,060] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:00,420] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:01,789] INFO : Plotted jets +rank0 [2026-10-28 11:50:02,833] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:43,642] INFO : Step 5100/50000 rank0 | Train Loss: 10.8364 | Train Loss Unweighted: 3.8688 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.4340 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1899 | Regression_eta: 0.0205 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1771 | Classification_binary: 3.4340 | Regression_cos_phi: 0.4539 | Regression_energy: 2.0810 | Regression_eta: 1.0092 | Regression_pt: 2.2213 | Regression_sin_phi: 0.4599 | LR: 9.77e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5826s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0030 | max_backward: 0.3092 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.4015 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3496.5 | uss_MB: 2718.6 | child_rss_MB: 4022.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:50:43,856] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:25,294] INFO : Step 5200/50000 rank0 | Train Loss: 10.6593 | Train Loss Unweighted: 3.7948 | Train Loss Components: Classification: 0.0238 | Classification_binary: 3.3521 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1983 | Regression_eta: 0.0136 | Regression_pt: 0.1909 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1062 | Classification_binary: 3.3521 | Regression_cos_phi: 0.5596 | Regression_energy: 2.1727 | Regression_eta: 0.6687 | Regression_pt: 2.2888 | Regression_sin_phi: 0.5113 | LR: 9.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7087s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1046 | max_loss: 0.0031 | max_backward: 0.3940 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3958 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3496.5 | uss_MB: 2720.8 | child_rss_MB: 4024.3 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:51:25,509] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:08,326] INFO : Step 5300/50000 rank0 | Train Loss: 10.0252 | Train Loss Unweighted: 3.5848 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.1807 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1791 | Regression_eta: 0.0127 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1755 | Classification_binary: 3.1807 | Regression_cos_phi: 0.5382 | Regression_energy: 1.9627 | Regression_eta: 0.6240 | Regression_pt: 2.0582 | Regression_sin_phi: 0.4859 | LR: 9.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5256s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1829 | max_loss: 0.0030 | max_backward: 0.2028 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3133 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4024.0 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:52:08,540] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:48,496] INFO : Step 5400/50000 rank0 | Train Loss: 10.6395 | Train Loss Unweighted: 3.5770 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.1260 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1995 | Regression_eta: 0.0168 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1187 | Classification_binary: 3.1260 | Regression_cos_phi: 0.5563 | Regression_energy: 2.1856 | Regression_eta: 0.8254 | Regression_pt: 2.3394 | Regression_sin_phi: 0.4882 | LR: 9.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4228s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1384 | max_loss: 0.0033 | max_backward: 0.2034 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4025.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:52:48,711] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:29,905] INFO : Step 5500/50000 rank0 | Train Loss: 10.4706 | Train Loss Unweighted: 3.5341 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1035 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1965 | Regression_eta: 0.0185 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1222 | Classification_binary: 3.1035 | Regression_cos_phi: 0.6030 | Regression_energy: 2.1528 | Regression_eta: 0.9077 | Regression_pt: 2.1024 | Regression_sin_phi: 0.4789 | LR: 9.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5663s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3082 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3847 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4021.1 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:53:30,118] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:10,887] INFO : Step 5600/50000 rank0 | Train Loss: 9.8073 | Train Loss Unweighted: 3.6007 | Train Loss Components: Classification: 0.0230 | Classification_binary: 3.2074 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1708 | Regression_eta: 0.0119 | Regression_pt: 0.1726 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0681 | Classification_binary: 3.2074 | Regression_cos_phi: 0.5402 | Regression_energy: 1.8711 | Regression_eta: 0.5841 | Regression_pt: 2.0695 | Regression_sin_phi: 0.4669 | LR: 9.72e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6146s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2986 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.4100 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4027.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:54:11,102] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:52,008] INFO : Step 5700/50000 rank0 | Train Loss: 9.9083 | Train Loss Unweighted: 3.5277 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.1307 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1762 | Regression_eta: 0.0141 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0706 | Classification_binary: 3.1307 | Regression_cos_phi: 0.5598 | Regression_energy: 1.9307 | Regression_eta: 0.6921 | Regression_pt: 2.0084 | Regression_sin_phi: 0.5158 | LR: 9.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4992s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2413 | max_optimizer_step: 0.1105 | max_tensorboard_logging: 0.3904 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3333.2 | uss_MB: 2555.9 | child_rss_MB: 4023.4 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:54:52,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:33,677] INFO : Step 5800/50000 rank0 | Train Loss: 10.3369 | Train Loss Unweighted: 3.5680 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1429 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1863 | Regression_eta: 0.0133 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1240 | Classification_binary: 3.1429 | Regression_cos_phi: 0.6385 | Regression_energy: 2.0416 | Regression_eta: 0.6559 | Regression_pt: 2.2035 | Regression_sin_phi: 0.5306 | LR: 9.70e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3583s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1040 | max_loss: 0.0032 | max_backward: 0.2029 | max_optimizer_step: 0.1078 | max_tensorboard_logging: 0.2815 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3333.2 | uss_MB: 2556.0 | child_rss_MB: 4026.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:55:33,893] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:14,468] INFO : Step 5900/50000 rank0 | Train Loss: 10.3648 | Train Loss Unweighted: 3.5715 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.1620 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1819 | Regression_eta: 0.0149 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1612 | Classification_binary: 3.1620 | Regression_cos_phi: 0.7267 | Regression_energy: 1.9936 | Regression_eta: 0.7315 | Regression_pt: 2.0176 | Regression_sin_phi: 0.5722 | LR: 9.69e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6689s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1038 | max_loss: 0.0029 | max_backward: 0.3941 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3932 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3333.2 | uss_MB: 2558.3 | child_rss_MB: 4026.6 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:56:14,680] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:55,836] INFO : Step 6000/50000 rank0 | Train Loss: 9.9408 | Train Loss Unweighted: 3.6323 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.2402 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1720 | Regression_eta: 0.0106 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1870 | Classification_binary: 3.2402 | Regression_cos_phi: 0.5826 | Regression_energy: 1.8846 | Regression_eta: 0.5224 | Regression_pt: 2.0087 | Regression_sin_phi: 0.5153 | LR: 9.68e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7770s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.2010 | max_loss: 0.0029 | max_backward: 0.2866 | max_optimizer_step: 0.1666 | max_tensorboard_logging: 0.3887 | max_tensorboard_flush: 0.0122 | Memory: rss_MB: 3333.2 | uss_MB: 2558.6 | child_rss_MB: 4024.6 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:56:56,050] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:56,069] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:56:56,242] INFO : Running validation on rank0 +rank0 [2026-10-28 11:56:56,242] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:56:56,242] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:56:57,477] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:56:57,621] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:56:57,752] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:56:57,826] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:56:57,895] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:56:57,963] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:56:58,032] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:56:58,101] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:56:58,207] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:56:58,336] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:56:58,467] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:56:58,538] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:56:58,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:56:58,676] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:56:58,744] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:56:58,813] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:56:58,922] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:56:58,930] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:56:58,933] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:56:58,935] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:56:58,936] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:56:58,939] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:56:58,940] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:56:58,943] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:56:58,946] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:56:58,949] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:56:58,951] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:56:58,954] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:56:58,955] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:56:58,958] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:56:58,961] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:56:58,962] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:57:06,514] INFO : VALIDATION | Step=6000/50000 | Train Loss=9.9408 | Valid Loss=10.1249 | Train Loss Unweighted=3.6323 | Valid Loss Unweighted=3.5942 | Stale=0 | ETA=326.7m +rank0 [2026-10-28 11:57:06,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:57:06,582] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:57:06,585] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:57:06,733] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:57:08,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:57:09,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:57:09,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:57:10,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:57:10,582] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:57:10,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:57:10,641] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:57:10,644] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:57:10,723] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:57:12,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:57:13,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:57:13,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:57:14,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:57:14,719] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:57:14,722] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:57:14,786] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:57:14,789] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:57:14,941] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:57:16,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:57:17,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:57:18,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:57:18,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:57:18,865] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:57:18,873] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:57:19,135] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:57:19,541] INFO : Plotted number of elements +rank0 [2026-10-28 11:57:21,426] INFO : Plotted jets +rank0 [2026-10-28 11:57:22,505] INFO : Plotted jet ratio +rank0 [2026-10-28 11:57:22,539] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:57:22,770] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:57:23,157] INFO : Plotted number of elements +rank0 [2026-10-28 11:57:24,543] INFO : Plotted jets +rank0 [2026-10-28 11:57:25,647] INFO : Plotted jet ratio +rank0 [2026-10-28 11:57:25,680] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:57:25,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:57:26,249] INFO : Plotted number of elements +rank0 [2026-10-28 11:57:27,619] INFO : Plotted jets +rank0 [2026-10-28 11:57:28,660] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:09,510] INFO : Step 6100/50000 rank0 | Train Loss: 9.9779 | Train Loss Unweighted: 3.5781 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.1667 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1824 | Regression_eta: 0.0120 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0901 | Classification_binary: 3.1667 | Regression_cos_phi: 0.5138 | Regression_energy: 1.9985 | Regression_eta: 0.5885 | Regression_pt: 2.1424 | Regression_sin_phi: 0.4779 | LR: 9.67e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.3098 | max_optimizer_step: 0.1154 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3555.8 | uss_MB: 2778.0 | child_rss_MB: 4017.9 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:58:09,721] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:50,616] INFO : Step 6200/50000 rank0 | Train Loss: 10.9314 | Train Loss Unweighted: 3.5342 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.1056 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1825 | Regression_eta: 0.0249 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1169 | Classification_binary: 3.1056 | Regression_cos_phi: 0.7573 | Regression_energy: 1.9996 | Regression_eta: 1.2265 | Regression_pt: 2.1185 | Regression_sin_phi: 0.6068 | LR: 9.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3841s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1115 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1099 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3555.8 | uss_MB: 2778.1 | child_rss_MB: 4016.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:58:50,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:32,103] INFO : Step 6300/50000 rank0 | Train Loss: 10.4803 | Train Loss Unweighted: 3.5221 | Train Loss Components: Classification: 0.0214 | Classification_binary: 3.1039 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1827 | Regression_eta: 0.0215 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9946 | Classification_binary: 3.1039 | Regression_cos_phi: 0.6916 | Regression_energy: 2.0019 | Regression_eta: 1.0579 | Regression_pt: 2.0885 | Regression_sin_phi: 0.5418 | LR: 9.65e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5643s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0034 | max_backward: 0.2775 | max_optimizer_step: 0.1132 | max_tensorboard_logging: 0.3927 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3555.8 | uss_MB: 2778.1 | child_rss_MB: 4021.3 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:59:32,318] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:13,325] INFO : Step 6400/50000 rank0 | Train Loss: 10.0738 | Train Loss Unweighted: 3.6327 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.2357 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1738 | Regression_eta: 0.0135 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0703 | Classification_binary: 3.2357 | Regression_cos_phi: 0.6410 | Regression_energy: 1.9049 | Regression_eta: 0.6655 | Regression_pt: 2.0274 | Regression_sin_phi: 0.5292 | LR: 9.64e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.5297s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0796 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.2937 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3555.8 | uss_MB: 2778.2 | child_rss_MB: 4018.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:00:13,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:54,088] INFO : Step 6500/50000 rank0 | Train Loss: 9.7882 | Train Loss Unweighted: 3.5004 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1178 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1683 | Regression_eta: 0.0164 | Regression_pt: 0.1589 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1220 | Classification_binary: 3.1178 | Regression_cos_phi: 0.5354 | Regression_energy: 1.8437 | Regression_eta: 0.8065 | Regression_pt: 1.9060 | Regression_sin_phi: 0.4568 | LR: 9.63e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3555.8 | uss_MB: 2778.2 | child_rss_MB: 4018.8 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:00:54,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:34,847] INFO : Step 6600/50000 rank0 | Train Loss: 10.0602 | Train Loss Unweighted: 3.5393 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.1406 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1751 | Regression_eta: 0.0165 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9895 | Classification_binary: 3.1406 | Regression_cos_phi: 0.5887 | Regression_energy: 1.9188 | Regression_eta: 0.8112 | Regression_pt: 2.0131 | Regression_sin_phi: 0.5983 | LR: 9.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6091s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3111 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.3911 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3555.8 | uss_MB: 2778.2 | child_rss_MB: 4018.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:01:35,058] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:16,187] INFO : Step 6700/50000 rank0 | Train Loss: 9.0442 | Train Loss Unweighted: 3.4562 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.0980 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1589 | Regression_eta: 0.0109 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9887 | Classification_binary: 3.0980 | Regression_cos_phi: 0.4421 | Regression_energy: 1.7409 | Regression_eta: 0.5382 | Regression_pt: 1.8558 | Regression_sin_phi: 0.3805 | LR: 9.61e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.4658s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.3114 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3407.6 | uss_MB: 2630.9 | child_rss_MB: 4019.9 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:02:16,400] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:56,654] INFO : Step 6800/50000 rank0 | Train Loss: 9.4863 | Train Loss Unweighted: 3.4043 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.0100 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1756 | Regression_eta: 0.0109 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9892 | Classification_binary: 3.0100 | Regression_cos_phi: 0.4640 | Regression_energy: 1.9240 | Regression_eta: 0.5348 | Regression_pt: 2.0621 | Regression_sin_phi: 0.5023 | LR: 9.60e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5202s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0811 | max_backward: 0.2025 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3832 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3407.6 | uss_MB: 2631.2 | child_rss_MB: 4021.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:02:56,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:37,543] INFO : Step 6900/50000 rank0 | Train Loss: 9.7951 | Train Loss Unweighted: 3.5275 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.1450 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1671 | Regression_eta: 0.0132 | Regression_pt: 0.1621 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0575 | Classification_binary: 3.1450 | Regression_cos_phi: 0.6984 | Regression_energy: 1.8307 | Regression_eta: 0.6476 | Regression_pt: 1.9435 | Regression_sin_phi: 0.4724 | LR: 9.58e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6345s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1042 | max_loss: 0.0030 | max_backward: 0.3796 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3407.6 | uss_MB: 2633.3 | child_rss_MB: 4023.1 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:03:37,757] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:18,528] INFO : Step 7000/50000 rank0 | Train Loss: 10.0575 | Train Loss Unweighted: 3.5061 | Train Loss Components: Classification: 0.0212 | Classification_binary: 3.0864 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1842 | Regression_eta: 0.0107 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9834 | Classification_binary: 3.0864 | Regression_cos_phi: 0.6562 | Regression_energy: 2.0182 | Regression_eta: 0.5261 | Regression_pt: 2.2219 | Regression_sin_phi: 0.5653 | LR: 9.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4492s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1851 | max_loss: 0.0032 | max_backward: 0.2036 | max_optimizer_step: 0.1097 | max_tensorboard_logging: 0.2879 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3407.6 | uss_MB: 2631.2 | child_rss_MB: 4023.0 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:04:18,741] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:18,760] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 12:04:18,936] INFO : Running validation on rank0 +rank0 [2026-10-28 12:04:18,937] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:04:18,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:04:20,259] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 12:04:20,404] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:04:20,534] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:04:20,608] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:04:20,677] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:04:20,745] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:04:20,815] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:04:20,883] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:04:20,990] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:04:21,120] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:04:21,251] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:04:21,321] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:04:21,390] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:04:21,457] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:04:21,526] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:04:21,596] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:04:21,706] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:04:21,712] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:04:21,716] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:04:21,719] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:04:21,721] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:04:21,722] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:04:21,725] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:04:21,728] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:04:21,732] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:04:21,734] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:04:21,735] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:04:21,738] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:04:21,741] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:04:21,742] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:04:21,745] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:04:21,748] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:04:29,344] INFO : VALIDATION | Step=7000/50000 | Train Loss=10.0575 | Valid Loss=9.6633 | Train Loss Unweighted=3.5061 | Valid Loss Unweighted=3.4446 | Stale=0 | ETA=319.0m +rank0 [2026-10-28 12:04:29,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:29,412] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:04:29,416] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:29,565] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:31,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:32,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:32,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:33,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:33,550] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:04:33,553] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:33,609] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:04:33,612] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:33,698] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:35,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:36,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:36,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:37,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:37,553] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:04:37,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:37,618] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:04:37,621] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:04:37,708] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:39,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:40,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:40,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:41,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:41,642] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:04:41,649] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:41,909] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:42,312] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:44,204] INFO : Plotted jets +rank0 [2026-10-28 12:04:45,284] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:45,318] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:45,549] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:45,939] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:47,325] INFO : Plotted jets +rank0 [2026-10-28 12:04:48,432] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:48,465] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:48,674] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:49,037] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:50,408] INFO : Plotted jets +rank0 [2026-10-28 12:04:51,445] INFO : Plotted jet ratio +rank0 [2026-10-28 12:05:33,252] INFO : Step 7100/50000 rank0 | Train Loss: 10.1219 | Train Loss Unweighted: 3.4971 | Train Loss Components: Classification: 0.0209 | Classification_binary: 3.0809 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1854 | Regression_eta: 0.0177 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9709 | Classification_binary: 3.0809 | Regression_cos_phi: 0.5381 | Regression_energy: 2.0313 | Regression_eta: 0.8706 | Regression_pt: 2.1143 | Regression_sin_phi: 0.5158 | LR: 9.56e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3042 | max_optimizer_step: 0.1156 | max_tensorboard_logging: 0.4048 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4025.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:05:33,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:14,934] INFO : Step 7200/50000 rank0 | Train Loss: 8.9809 | Train Loss Unweighted: 3.4482 | Train Loss Components: Classification: 0.0208 | Classification_binary: 3.0948 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1561 | Regression_eta: 0.0098 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9630 | Classification_binary: 3.0948 | Regression_cos_phi: 0.4916 | Regression_energy: 1.7109 | Regression_eta: 0.4805 | Regression_pt: 1.8387 | Regression_sin_phi: 0.4014 | LR: 9.55e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4672s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1830 | max_loss: 0.0031 | max_backward: 0.2023 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4025.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:06:15,147] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:55,739] INFO : Step 7300/50000 rank0 | Train Loss: 9.7861 | Train Loss Unweighted: 3.4673 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0670 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1746 | Regression_eta: 0.0122 | Regression_pt: 0.1751 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0302 | Classification_binary: 3.0670 | Regression_cos_phi: 0.5820 | Regression_energy: 1.9136 | Regression_eta: 0.6009 | Regression_pt: 2.0999 | Regression_sin_phi: 0.4924 | LR: 9.53e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5332s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0030 | max_backward: 0.2624 | max_optimizer_step: 0.1128 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4020.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:06:55,955] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:36,885] INFO : Step 7400/50000 rank0 | Train Loss: 10.5199 | Train Loss Unweighted: 3.5094 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0572 | Regression_cos_phi: 0.0077 | Regression_energy: 0.2018 | Regression_eta: 0.0181 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0190 | Classification_binary: 3.0572 | Regression_cos_phi: 0.5445 | Regression_energy: 2.2113 | Regression_eta: 0.8896 | Regression_pt: 2.3426 | Regression_sin_phi: 0.4557 | LR: 9.52e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5008s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0458 | max_forward: 0.0772 | max_loss: 0.0031 | max_backward: 0.2021 | max_optimizer_step: 0.1826 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4020.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:07:37,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:17,453] INFO : Step 7500/50000 rank0 | Train Loss: 10.3226 | Train Loss Unweighted: 3.5274 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.1113 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1820 | Regression_eta: 0.0163 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1562 | Classification_binary: 3.1113 | Regression_cos_phi: 0.6400 | Regression_energy: 1.9945 | Regression_eta: 0.8017 | Regression_pt: 2.1050 | Regression_sin_phi: 0.5139 | LR: 9.51e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5527s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0990 | max_backward: 0.2025 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4020.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:08:17,664] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:59,104] INFO : Step 7600/50000 rank0 | Train Loss: 10.8716 | Train Loss Unweighted: 3.6769 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.2321 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1977 | Regression_eta: 0.0202 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0250 | Classification_binary: 3.2321 | Regression_cos_phi: 0.6214 | Regression_energy: 2.1667 | Regression_eta: 0.9914 | Regression_pt: 2.2373 | Regression_sin_phi: 0.5978 | LR: 9.50e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5349s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0435 | max_backward: 0.2020 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4021.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:08:59,316] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:40,620] INFO : Step 7700/50000 rank0 | Train Loss: 9.6674 | Train Loss Unweighted: 3.3850 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.9820 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1789 | Regression_eta: 0.0112 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0125 | Classification_binary: 2.9820 | Regression_cos_phi: 0.5771 | Regression_energy: 1.9599 | Regression_eta: 0.5493 | Regression_pt: 2.1027 | Regression_sin_phi: 0.4839 | LR: 9.48e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5488s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0031 | max_backward: 0.3094 | max_optimizer_step: 0.1115 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3484.3 | uss_MB: 2706.0 | child_rss_MB: 4024.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:09:40,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:22,352] INFO : Step 7800/50000 rank0 | Train Loss: 9.8552 | Train Loss Unweighted: 3.5135 | Train Loss Components: Classification: 0.0210 | Classification_binary: 3.1000 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1842 | Regression_eta: 0.0118 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9735 | Classification_binary: 3.1000 | Regression_cos_phi: 0.5362 | Regression_energy: 2.0181 | Regression_eta: 0.5780 | Regression_pt: 2.1760 | Regression_sin_phi: 0.4735 | LR: 9.47e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5411s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0927 | max_backward: 0.2023 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3485.3 | uss_MB: 2706.3 | child_rss_MB: 4024.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:10:22,564] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:03,199] INFO : Step 7900/50000 rank0 | Train Loss: 9.6907 | Train Loss Unweighted: 3.4372 | Train Loss Components: Classification: 0.0212 | Classification_binary: 3.0498 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1738 | Regression_eta: 0.0162 | Regression_pt: 0.1607 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9858 | Classification_binary: 3.0498 | Regression_cos_phi: 0.4973 | Regression_energy: 1.9047 | Regression_eta: 0.7964 | Regression_pt: 1.9267 | Regression_sin_phi: 0.5300 | LR: 9.46e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4987s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0687 | max_backward: 0.2022 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3485.3 | uss_MB: 2706.4 | child_rss_MB: 4021.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:11:03,414] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:44,369] INFO : Step 8000/50000 rank0 | Train Loss: 9.3785 | Train Loss Unweighted: 3.3889 | Train Loss Components: Classification: 0.0226 | Classification_binary: 3.0202 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1602 | Regression_eta: 0.0107 | Regression_pt: 0.1584 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0503 | Classification_binary: 3.0202 | Regression_cos_phi: 0.6913 | Regression_energy: 1.7558 | Regression_eta: 0.5244 | Regression_pt: 1.8994 | Regression_sin_phi: 0.4370 | LR: 9.44e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.4318s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3080 | max_optimizer_step: 0.1100 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3485.3 | uss_MB: 2706.5 | child_rss_MB: 4044.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:11:44,582] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:44,600] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 12:11:44,773] INFO : Running validation on rank0 +rank0 [2026-10-28 12:11:44,773] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:11:44,773] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:11:46,024] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 12:11:46,167] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:11:46,299] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:11:46,373] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:11:46,442] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:11:46,513] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:11:46,582] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:11:46,651] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:11:46,760] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:11:46,890] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:11:47,020] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:11:47,091] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:11:47,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:11:47,228] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:11:47,297] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:11:47,366] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:11:47,475] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:11:47,482] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:11:47,485] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:11:47,488] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:11:47,489] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:11:47,492] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:11:47,495] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:11:47,498] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:11:47,500] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:11:47,501] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:11:47,504] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:11:47,506] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:11:47,507] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:11:47,510] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:11:47,512] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:11:47,514] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:11:55,089] INFO : VALIDATION | Step=8000/50000 | Train Loss=9.3785 | Valid Loss=9.5710 | Train Loss Unweighted=3.3889 | Valid Loss Unweighted=3.4009 | Stale=0 | ETA=311.7m +rank0 [2026-10-28 12:11:55,098] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:55,158] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:11:55,162] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:11:55,310] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:57,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:58,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:58,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:59,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:59,497] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:11:59,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:59,556] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:11:59,559] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:11:59,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:01,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:02,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:02,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:03,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:03,595] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:12:03,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:03,653] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:12:03,656] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:12:03,812] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:05,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:06,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:06,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:07,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:07,697] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:12:07,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:07,961] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:08,364] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:10,277] INFO : Plotted jets +rank0 [2026-10-28 12:12:11,359] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:11,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:11,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:12,018] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:13,410] INFO : Plotted jets +rank0 [2026-10-28 12:12:14,508] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:14,540] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 12:12:14,747] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 12:12:15,108] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:16,479] INFO : Plotted jets +rank0 [2026-10-28 12:12:17,520] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:58,909] INFO : Step 8100/50000 rank0 | Train Loss: 9.3781 | Train Loss Unweighted: 3.2862 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9150 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1632 | Regression_eta: 0.0146 | Regression_pt: 0.1554 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0052 | Classification_binary: 2.9150 | Regression_cos_phi: 0.5202 | Regression_energy: 1.7879 | Regression_eta: 0.7199 | Regression_pt: 1.8633 | Regression_sin_phi: 0.5665 | LR: 9.43e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1093 | max_forward: 0.1039 | max_loss: 0.0031 | max_backward: 0.3937 | max_optimizer_step: 0.1145 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3684.9 | uss_MB: 2912.4 | child_rss_MB: 4045.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:12:59,123] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:39,905] INFO : Step 8200/50000 rank0 | Train Loss: 9.3862 | Train Loss Unweighted: 3.4096 | Train Loss Components: Classification: 0.0196 | Classification_binary: 3.0381 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1628 | Regression_eta: 0.0158 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9092 | Classification_binary: 3.0381 | Regression_cos_phi: 0.4944 | Regression_energy: 1.7833 | Regression_eta: 0.7755 | Regression_pt: 1.9044 | Regression_sin_phi: 0.4812 | LR: 9.42e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5903s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0033 | max_backward: 0.3101 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3684.9 | uss_MB: 2910.4 | child_rss_MB: 4017.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:13:40,119] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:20,833] INFO : Step 8300/50000 rank0 | Train Loss: 9.1849 | Train Loss Unweighted: 3.2781 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.9016 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1674 | Regression_eta: 0.0107 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9062 | Classification_binary: 2.9016 | Regression_cos_phi: 0.6362 | Regression_energy: 1.8344 | Regression_eta: 0.5255 | Regression_pt: 1.9566 | Regression_sin_phi: 0.4243 | LR: 9.40e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.2266 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3684.9 | uss_MB: 2910.5 | child_rss_MB: 4019.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:14:21,047] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:02,467] INFO : Step 8400/50000 rank0 | Train Loss: 9.2624 | Train Loss Unweighted: 3.4558 | Train Loss Components: Classification: 0.0202 | Classification_binary: 3.0929 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1600 | Regression_eta: 0.0125 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9395 | Classification_binary: 3.0929 | Regression_cos_phi: 0.5417 | Regression_energy: 1.7526 | Regression_eta: 0.6138 | Regression_pt: 1.8622 | Regression_sin_phi: 0.4596 | LR: 9.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.2927 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3964 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3684.9 | uss_MB: 2910.5 | child_rss_MB: 4019.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:15:02,683] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:43,243] INFO : Step 8500/50000 rank0 | Train Loss: 9.6020 | Train Loss Unweighted: 3.4346 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0506 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1685 | Regression_eta: 0.0116 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0574 | Classification_binary: 3.0506 | Regression_cos_phi: 0.5681 | Regression_energy: 1.8465 | Regression_eta: 0.5686 | Regression_pt: 1.9739 | Regression_sin_phi: 0.5368 | LR: 9.37e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6650s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3105 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.4194 | max_tensorboard_flush: 0.0134 | Memory: rss_MB: 3685.9 | uss_MB: 2910.5 | child_rss_MB: 4018.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:15:43,458] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:24,238] INFO : Step 8600/50000 rank0 | Train Loss: 9.2516 | Train Loss Unweighted: 3.3888 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.0301 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1559 | Regression_eta: 0.0124 | Regression_pt: 0.1527 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0053 | Classification_binary: 3.0301 | Regression_cos_phi: 0.5530 | Regression_energy: 1.7083 | Regression_eta: 0.6089 | Regression_pt: 1.8309 | Regression_sin_phi: 0.5150 | LR: 9.36e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0992 | max_loss: 0.0032 | max_backward: 0.2025 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3156 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3685.9 | uss_MB: 2910.5 | child_rss_MB: 4020.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:16:24,457] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:05,328] INFO : Step 8700/50000 rank0 | Train Loss: 9.8021 | Train Loss Unweighted: 3.3609 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9745 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1702 | Regression_eta: 0.0173 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0138 | Classification_binary: 2.9745 | Regression_cos_phi: 0.6777 | Regression_energy: 1.8647 | Regression_eta: 0.8485 | Regression_pt: 1.9118 | Regression_sin_phi: 0.5110 | LR: 9.34e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4798s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0031 | max_backward: 0.3092 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2976 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3554.2 | uss_MB: 2778.7 | child_rss_MB: 4045.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:17:05,542] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:46,398] INFO : Step 8800/50000 rank0 | Train Loss: 9.1497 | Train Loss Unweighted: 3.4013 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0462 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1557 | Regression_eta: 0.0111 | Regression_pt: 0.1508 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0584 | Classification_binary: 3.0462 | Regression_cos_phi: 0.5452 | Regression_energy: 1.7061 | Regression_eta: 0.5459 | Regression_pt: 1.8081 | Regression_sin_phi: 0.4399 | LR: 9.33e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5457s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0033 | max_backward: 0.3094 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3554.2 | uss_MB: 2778.6 | child_rss_MB: 4013.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:17:46,612] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:26,962] INFO : Step 8900/50000 rank0 | Train Loss: 9.3941 | Train Loss Unweighted: 3.3843 | Train Loss Components: Classification: 0.0202 | Classification_binary: 3.0078 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1658 | Regression_eta: 0.0119 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9387 | Classification_binary: 3.0078 | Regression_cos_phi: 0.5930 | Regression_energy: 1.8165 | Regression_eta: 0.5848 | Regression_pt: 1.9449 | Regression_sin_phi: 0.5084 | LR: 9.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4546s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0226 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.2023 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3554.2 | uss_MB: 2778.4 | child_rss_MB: 4017.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 12:18:27,193] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:07,606] INFO : Step 9000/50000 rank0 | Train Loss: 8.6657 | Train Loss Unweighted: 3.1986 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.8525 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1518 | Regression_eta: 0.0085 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0267 | Classification_binary: 2.8525 | Regression_cos_phi: 0.5075 | Regression_energy: 1.6629 | Regression_eta: 0.4169 | Regression_pt: 1.8007 | Regression_sin_phi: 0.3985 | LR: 9.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5433s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0833 | max_backward: 0.2028 | max_optimizer_step: 0.1103 | max_tensorboard_logging: 0.4005 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3554.2 | uss_MB: 2778.5 | child_rss_MB: 4017.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:19:07,820] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:07,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 12:19:08,029] INFO : Running validation on rank0 +rank0 [2026-10-28 12:19:08,029] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:19:08,029] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:19:09,363] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 12:19:09,507] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:19:09,638] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:19:09,712] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:19:09,781] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:19:09,849] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:19:09,919] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:19:09,988] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:19:10,098] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:19:10,229] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:19:10,359] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:19:10,430] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:19:10,498] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:19:10,567] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:19:10,635] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:19:10,704] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:19:10,807] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:19:10,814] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:19:10,817] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:19:10,820] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:19:10,821] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:19:10,824] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:19:10,826] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:19:10,828] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:19:10,831] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:19:10,834] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:19:10,836] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:19:10,838] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:19:10,842] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:19:10,844] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:19:10,846] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:19:10,849] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:19:18,318] INFO : VALIDATION | Step=9000/50000 | Train Loss=8.6657 | Valid Loss=9.5121 | Train Loss Unweighted=3.1986 | Valid Loss Unweighted=3.3604 | Stale=0 | ETA=304.1m +rank0 [2026-10-28 12:19:18,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:18,415] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:19:18,418] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:19:18,662] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:20,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:21,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:22,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:22,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:22,730] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:19:22,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:22,788] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:19:22,791] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:22,876] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:24,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:25,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:25,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:26,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:26,757] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:19:26,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:26,836] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:19:26,839] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:19:26,915] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:28,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:29,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:30,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:30,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:30,871] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:19:30,877] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,134] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,539] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:33,463] INFO : Plotted jets +rank0 [2026-10-28 12:19:34,544] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:34,576] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:34,809] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:35,196] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:36,585] INFO : Plotted jets +rank0 [2026-10-28 12:19:37,689] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:37,722] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 12:19:37,930] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 12:19:38,294] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:39,667] INFO : Plotted jets +rank0 [2026-10-28 12:19:40,711] INFO : Plotted jet ratio +rank0 [2026-10-28 12:20:21,554] INFO : Step 9100/50000 rank0 | Train Loss: 9.2302 | Train Loss Unweighted: 3.3404 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.9630 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1657 | Regression_eta: 0.0112 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9607 | Classification_binary: 2.9630 | Regression_cos_phi: 0.5087 | Regression_energy: 1.8154 | Regression_eta: 0.5522 | Regression_pt: 1.9848 | Regression_sin_phi: 0.4455 | LR: 9.28e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5622s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1857 | max_loss: 0.0033 | max_backward: 0.2023 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3364 | max_tensorboard_flush: 0.0151 | Memory: rss_MB: 3625.5 | uss_MB: 2850.6 | child_rss_MB: 4016.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:20:21,776] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:02,602] INFO : Step 9200/50000 rank0 | Train Loss: 10.4475 | Train Loss Unweighted: 3.4601 | Train Loss Components: Classification: 0.0216 | Classification_binary: 3.0262 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1927 | Regression_eta: 0.0201 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0026 | Classification_binary: 3.0262 | Regression_cos_phi: 0.6756 | Regression_energy: 2.1110 | Regression_eta: 0.9889 | Regression_pt: 2.1930 | Regression_sin_phi: 0.4501 | LR: 9.27e-04 | DataLoad Time: 0.0034s | Model Forward Time: 1.5510s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0618 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2023 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 0.3111 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3627.5 | uss_MB: 2852.1 | child_rss_MB: 4016.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:21:02,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:44,018] INFO : Step 9300/50000 rank0 | Train Loss: 9.1003 | Train Loss Unweighted: 3.2582 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.8962 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1585 | Regression_eta: 0.0151 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9310 | Classification_binary: 2.8962 | Regression_cos_phi: 0.4409 | Regression_energy: 1.7362 | Regression_eta: 0.7405 | Regression_pt: 1.8489 | Regression_sin_phi: 0.5067 | LR: 9.25e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5364s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1816 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3189 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3627.5 | uss_MB: 2852.3 | child_rss_MB: 4016.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:21:44,234] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:25,642] INFO : Step 9400/50000 rank0 | Train Loss: 10.2354 | Train Loss Unweighted: 3.5125 | Train Loss Components: Classification: 0.0196 | Classification_binary: 3.1231 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1696 | Regression_eta: 0.0239 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9089 | Classification_binary: 3.1231 | Regression_cos_phi: 0.6816 | Regression_energy: 1.8583 | Regression_eta: 1.1761 | Regression_pt: 1.8824 | Regression_sin_phi: 0.6050 | LR: 9.24e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5919s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0636 | max_backward: 0.2023 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.3878 | max_tensorboard_flush: 0.0128 | Memory: rss_MB: 3627.5 | uss_MB: 2852.4 | child_rss_MB: 4015.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:22:25,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:06,786] INFO : Step 9500/50000 rank0 | Train Loss: 8.5242 | Train Loss Unweighted: 3.1861 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.8469 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1501 | Regression_eta: 0.0084 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9586 | Classification_binary: 2.8469 | Regression_cos_phi: 0.4932 | Regression_energy: 1.6448 | Regression_eta: 0.4139 | Regression_pt: 1.7573 | Regression_sin_phi: 0.4095 | LR: 9.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5502s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0409 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.2026 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.3144 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 3627.5 | uss_MB: 2852.4 | child_rss_MB: 4019.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:23:07,000] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:48,785] INFO : Step 9600/50000 rank0 | Train Loss: 8.7320 | Train Loss Unweighted: 3.2264 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.8762 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1556 | Regression_eta: 0.0106 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9608 | Classification_binary: 2.8762 | Regression_cos_phi: 0.4732 | Regression_energy: 1.7046 | Regression_eta: 0.5227 | Regression_pt: 1.8026 | Regression_sin_phi: 0.3919 | LR: 9.21e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5900s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.3028 | max_optimizer_step: 0.1171 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3627.5 | uss_MB: 2852.4 | child_rss_MB: 4018.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:23:48,999] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:30,492] INFO : Step 9700/50000 rank0 | Train Loss: 8.9081 | Train Loss Unweighted: 3.2550 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.8900 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1618 | Regression_eta: 0.0105 | Regression_pt: 0.1589 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9888 | Classification_binary: 2.8900 | Regression_cos_phi: 0.4276 | Regression_energy: 1.7729 | Regression_eta: 0.5159 | Regression_pt: 1.9051 | Regression_sin_phi: 0.4077 | LR: 9.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5805s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0032 | max_backward: 0.3092 | max_optimizer_step: 0.1132 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3627.0 | uss_MB: 2850.5 | child_rss_MB: 4015.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:24:30,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:12,200] INFO : Step 9800/50000 rank0 | Train Loss: 8.9568 | Train Loss Unweighted: 3.2080 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8562 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1559 | Regression_eta: 0.0141 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9433 | Classification_binary: 2.8562 | Regression_cos_phi: 0.5647 | Regression_energy: 1.7085 | Regression_eta: 0.6925 | Regression_pt: 1.7583 | Regression_sin_phi: 0.4333 | LR: 9.17e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4842s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0286 | max_backward: 0.2027 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.3781 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3627.0 | uss_MB: 2852.0 | child_rss_MB: 4017.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:25:12,412] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:52,681] INFO : Step 9900/50000 rank0 | Train Loss: 9.9009 | Train Loss Unweighted: 3.4068 | Train Loss Components: Classification: 0.0209 | Classification_binary: 3.0022 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1780 | Regression_eta: 0.0125 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9681 | Classification_binary: 3.0022 | Regression_cos_phi: 0.5704 | Regression_energy: 1.9502 | Regression_eta: 0.6137 | Regression_pt: 2.0861 | Regression_sin_phi: 0.7102 | LR: 9.16e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6715s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2127 | max_loss: 0.0034 | max_backward: 0.2861 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.2856 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3627.0 | uss_MB: 2854.6 | child_rss_MB: 4014.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:25:52,912] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:33,223] INFO : Step 10000/50000 rank0 | Train Loss: 9.4590 | Train Loss Unweighted: 3.3266 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9277 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1790 | Regression_eta: 0.0100 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9832 | Classification_binary: 2.9277 | Regression_cos_phi: 0.5710 | Regression_energy: 1.9618 | Regression_eta: 0.4929 | Regression_pt: 2.0818 | Regression_sin_phi: 0.4406 | LR: 9.14e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6978s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3937 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3909 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3627.0 | uss_MB: 2854.7 | child_rss_MB: 4013.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:26:33,437] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:33,457] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 12:26:33,733] INFO : Running validation on rank0 +rank0 [2026-10-28 12:26:33,733] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:26:33,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:26:35,039] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 12:26:35,185] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:26:35,316] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:26:35,391] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:26:35,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:26:35,528] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:26:35,598] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:26:35,667] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:26:35,775] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:26:35,907] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:26:36,038] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:26:36,110] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:26:36,179] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:26:36,247] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:26:36,316] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:26:36,385] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:26:36,488] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:26:36,494] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:26:36,497] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:26:36,500] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:26:36,501] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:26:36,504] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:26:36,505] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:26:36,508] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:26:36,510] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:26:36,514] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:26:36,515] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:26:36,517] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:26:36,518] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:26:36,520] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:26:36,521] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:26:36,524] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:26:43,996] INFO : VALIDATION | Step=10000/50000 | Train Loss=9.4590 | Valid Loss=9.1721 | Train Loss Unweighted=3.3266 | Valid Loss Unweighted=3.2697 | Stale=0 | ETA=296.7m +rank0 [2026-10-28 12:26:44,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:44,076] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:26:44,080] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:26:44,285] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:46,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:46,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:47,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:48,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:48,401] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:26:48,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:48,472] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:26:48,475] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:48,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:50,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:51,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:51,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:52,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:52,477] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:26:52,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:52,568] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:26:52,571] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:26:52,637] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:54,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:55,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:55,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:56,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:56,581] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:26:56,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:26:56,843] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:26:57,247] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:59,151] INFO : Plotted jets +rank0 [2026-10-28 12:27:00,226] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:00,259] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:00,491] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:00,879] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:02,259] INFO : Plotted jets +rank0 [2026-10-28 12:27:03,362] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:03,395] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:03,602] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:03,966] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:05,341] INFO : Plotted jets +rank0 [2026-10-28 12:27:06,389] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:47,598] INFO : Step 10100/50000 rank0 | Train Loss: 9.3153 | Train Loss Unweighted: 3.3198 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9753 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1455 | Regression_eta: 0.0177 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0975 | Classification_binary: 2.9753 | Regression_cos_phi: 0.6233 | Regression_energy: 1.5939 | Regression_eta: 0.8685 | Regression_pt: 1.6989 | Regression_sin_phi: 0.4580 | LR: 9.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6437s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3103 | max_optimizer_step: 0.1215 | max_tensorboard_logging: 0.4122 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3699.6 | uss_MB: 2924.9 | child_rss_MB: 4008.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:27:47,812] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:29,152] INFO : Step 10200/50000 rank0 | Train Loss: 9.2086 | Train Loss Unweighted: 3.2862 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9299 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1531 | Regression_eta: 0.0158 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0409 | Classification_binary: 2.9299 | Regression_cos_phi: 0.5289 | Regression_energy: 1.6779 | Regression_eta: 0.7771 | Regression_pt: 1.8017 | Regression_sin_phi: 0.4521 | LR: 9.11e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5240s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2435 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3787 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3700.6 | uss_MB: 2926.5 | child_rss_MB: 4005.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:28:29,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:09,728] INFO : Step 10300/50000 rank0 | Train Loss: 10.6029 | Train Loss Unweighted: 3.4198 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.9795 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1933 | Regression_eta: 0.0238 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9965 | Classification_binary: 2.9795 | Regression_cos_phi: 0.5911 | Regression_energy: 2.1185 | Regression_eta: 1.1683 | Regression_pt: 2.2167 | Regression_sin_phi: 0.5323 | LR: 9.09e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5208s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1790 | max_loss: 0.0030 | max_backward: 0.2024 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2999 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3700.6 | uss_MB: 2926.6 | child_rss_MB: 4009.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:29:09,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:51,402] INFO : Step 10400/50000 rank0 | Train Loss: 8.9836 | Train Loss Unweighted: 3.2755 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.9044 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1650 | Regression_eta: 0.0087 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0021 | Classification_binary: 2.9044 | Regression_cos_phi: 0.4515 | Regression_energy: 1.8080 | Regression_eta: 0.4266 | Regression_pt: 1.9478 | Regression_sin_phi: 0.4432 | LR: 9.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.2580 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3864 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3700.6 | uss_MB: 2926.6 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:29:51,616] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:32,417] INFO : Step 10500/50000 rank0 | Train Loss: 9.2066 | Train Loss Unweighted: 3.2772 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9076 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1616 | Regression_eta: 0.0109 | Regression_pt: 0.1584 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0956 | Classification_binary: 2.9076 | Regression_cos_phi: 0.5005 | Regression_energy: 1.7706 | Regression_eta: 0.5371 | Regression_pt: 1.9000 | Regression_sin_phi: 0.4951 | LR: 9.06e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3107 | max_optimizer_step: 0.1207 | max_tensorboard_logging: 0.2773 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3700.6 | uss_MB: 2926.5 | child_rss_MB: 4011.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:30:32,630] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:13,494] INFO : Step 10600/50000 rank0 | Train Loss: 9.6617 | Train Loss Unweighted: 3.3332 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9353 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1764 | Regression_eta: 0.0145 | Regression_pt: 0.1699 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9827 | Classification_binary: 2.9353 | Regression_cos_phi: 0.5741 | Regression_energy: 1.9329 | Regression_eta: 0.7139 | Regression_pt: 2.0373 | Regression_sin_phi: 0.4854 | LR: 9.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5721s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2834 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3844 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3700.6 | uss_MB: 2926.7 | child_rss_MB: 4011.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:31:13,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:54,887] INFO : Step 10700/50000 rank0 | Train Loss: 9.7603 | Train Loss Unweighted: 3.3193 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9321 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1701 | Regression_eta: 0.0195 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9828 | Classification_binary: 2.9321 | Regression_cos_phi: 0.5325 | Regression_energy: 1.8643 | Regression_eta: 0.9591 | Regression_pt: 1.9154 | Regression_sin_phi: 0.5741 | LR: 9.02e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4991s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1811 | max_loss: 0.0031 | max_backward: 0.2023 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.2991 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3700.1 | uss_MB: 2925.0 | child_rss_MB: 4013.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:31:55,099] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:36,749] INFO : Step 10800/50000 rank0 | Train Loss: 9.2213 | Train Loss Unweighted: 3.2397 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.8608 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1670 | Regression_eta: 0.0092 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0908 | Classification_binary: 2.8608 | Regression_cos_phi: 0.5574 | Regression_energy: 1.8302 | Regression_eta: 0.4543 | Regression_pt: 1.9643 | Regression_sin_phi: 0.4634 | LR: 9.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5933s | Phase Times: max_pre_step_barrier: 0.1051 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.3061 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3701.1 | uss_MB: 2926.6 | child_rss_MB: 4034.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:32:36,964] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:18,470] INFO : Step 10900/50000 rank0 | Train Loss: 8.5178 | Train Loss Unweighted: 3.2527 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.9138 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1507 | Regression_eta: 0.0087 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9381 | Classification_binary: 2.9138 | Regression_cos_phi: 0.4330 | Regression_energy: 1.6517 | Regression_eta: 0.4277 | Regression_pt: 1.7622 | Regression_sin_phi: 0.3913 | LR: 8.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5844s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3058 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.3897 | max_tensorboard_flush: 0.0056 | Memory: rss_MB: 3701.1 | uss_MB: 2926.6 | child_rss_MB: 4016.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:33:18,686] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:59,841] INFO : Step 11000/50000 rank0 | Train Loss: 8.5435 | Train Loss Unweighted: 3.2313 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8933 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1511 | Regression_eta: 0.0099 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9237 | Classification_binary: 2.8933 | Regression_cos_phi: 0.4400 | Regression_energy: 1.6553 | Regression_eta: 0.4870 | Regression_pt: 1.7307 | Regression_sin_phi: 0.4136 | LR: 8.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5330s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0831 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2022 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3701.1 | uss_MB: 2926.7 | child_rss_MB: 4016.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:34:00,075] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:00,094] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 12:34:00,270] INFO : Running validation on rank0 +rank0 [2026-10-28 12:34:00,271] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:34:00,271] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:34:01,583] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 12:34:01,729] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:34:01,861] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:34:01,937] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:34:02,007] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:34:02,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:34:02,146] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:34:02,215] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:34:02,324] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:34:02,455] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:34:02,586] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:34:02,657] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:34:02,726] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:34:02,794] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:34:02,864] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:34:02,933] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:34:03,051] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:34:03,058] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:34:03,061] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:34:03,064] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:34:03,065] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:34:03,068] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:34:03,070] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:34:03,072] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:34:03,074] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:34:03,078] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:34:03,080] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:34:03,081] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:34:03,084] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:34:03,086] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:34:03,089] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:34:03,090] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:34:10,668] INFO : VALIDATION | Step=11000/50000 | Train Loss=8.5435 | Valid Loss=8.9201 | Train Loss Unweighted=3.2313 | Valid Loss Unweighted=3.1908 | Stale=0 | ETA=289.4m +rank0 [2026-10-28 12:34:10,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:10,735] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:34:10,739] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:10,893] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:12,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:13,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:14,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:15,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:15,103] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:34:15,106] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:15,161] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:34:15,164] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:15,262] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:17,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:17,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:18,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:19,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:19,241] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:34:19,244] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:19,298] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:34:19,302] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:34:19,403] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:21,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:21,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:22,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:23,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:23,360] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:34:23,367] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:23,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:24,026] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:25,922] INFO : Plotted jets +rank0 [2026-10-28 12:34:27,001] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:27,034] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:27,265] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:27,657] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:29,048] INFO : Plotted jets +rank0 [2026-10-28 12:34:30,157] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:30,190] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:30,399] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:30,759] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:32,142] INFO : Plotted jets +rank0 [2026-10-28 12:34:33,200] INFO : Plotted jet ratio +rank0 [2026-10-28 12:35:14,348] INFO : Step 11100/50000 rank0 | Train Loss: 8.9500 | Train Loss Unweighted: 3.0962 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7295 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1615 | Regression_eta: 0.0120 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9195 | Classification_binary: 2.7295 | Regression_cos_phi: 0.5917 | Regression_energy: 1.7695 | Regression_eta: 0.5893 | Regression_pt: 1.8917 | Regression_sin_phi: 0.4588 | LR: 8.95e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8261s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0879 | max_forward: 0.1040 | max_loss: 0.0031 | max_backward: 0.3934 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3003 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3771.7 | uss_MB: 3001.9 | child_rss_MB: 4015.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:35:14,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:56,085] INFO : Step 11200/50000 rank0 | Train Loss: 9.3948 | Train Loss Unweighted: 3.2278 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8492 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1662 | Regression_eta: 0.0192 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9224 | Classification_binary: 2.8492 | Regression_cos_phi: 0.5050 | Regression_energy: 1.8213 | Regression_eta: 0.9445 | Regression_pt: 1.9080 | Regression_sin_phi: 0.4444 | LR: 8.93e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5850s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0014 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.3096 | max_optimizer_step: 0.1150 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3773.7 | uss_MB: 3001.4 | child_rss_MB: 4014.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:35:56,300] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:36,433] INFO : Step 11300/50000 rank0 | Train Loss: 8.5869 | Train Loss Unweighted: 3.1868 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.8434 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1518 | Regression_eta: 0.0085 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9994 | Classification_binary: 2.8434 | Regression_cos_phi: 0.5066 | Regression_energy: 1.6637 | Regression_eta: 0.4194 | Regression_pt: 1.7782 | Regression_sin_phi: 0.3762 | LR: 8.91e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1028 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3773.7 | uss_MB: 3001.7 | child_rss_MB: 4014.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:36:36,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:17,850] INFO : Step 11400/50000 rank0 | Train Loss: 10.0288 | Train Loss Unweighted: 3.4148 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9918 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1905 | Regression_eta: 0.0127 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0384 | Classification_binary: 2.9918 | Regression_cos_phi: 0.6083 | Regression_energy: 2.0872 | Regression_eta: 0.6250 | Regression_pt: 2.1674 | Regression_sin_phi: 0.5107 | LR: 8.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5586s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2683 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3773.7 | uss_MB: 3001.7 | child_rss_MB: 4008.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:37:18,063] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:58,745] INFO : Step 11500/50000 rank0 | Train Loss: 9.4975 | Train Loss Unweighted: 3.3741 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9963 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1641 | Regression_eta: 0.0149 | Regression_pt: 0.1619 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9843 | Classification_binary: 2.9963 | Regression_cos_phi: 0.5135 | Regression_energy: 1.7986 | Regression_eta: 0.7316 | Regression_pt: 1.9419 | Regression_sin_phi: 0.5313 | LR: 8.88e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.3069 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3843 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3773.7 | uss_MB: 3001.8 | child_rss_MB: 4011.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:37:58,959] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:38,960] INFO : Step 11600/50000 rank0 | Train Loss: 9.3977 | Train Loss Unweighted: 3.3901 | Train Loss Components: Classification: 0.0211 | Classification_binary: 3.0019 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1749 | Regression_eta: 0.0108 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9810 | Classification_binary: 3.0019 | Regression_cos_phi: 0.4921 | Regression_energy: 1.9162 | Regression_eta: 0.5291 | Regression_pt: 2.0010 | Regression_sin_phi: 0.4764 | LR: 8.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5539s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0913 | max_backward: 0.2026 | max_optimizer_step: 0.1125 | max_tensorboard_logging: 0.3802 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3773.7 | uss_MB: 3002.0 | child_rss_MB: 4011.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:38:39,174] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:20,497] INFO : Step 11700/50000 rank0 | Train Loss: 9.0855 | Train Loss Unweighted: 3.1975 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.8451 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1543 | Regression_eta: 0.0172 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0000 | Classification_binary: 2.8451 | Regression_cos_phi: 0.5402 | Regression_energy: 1.6902 | Regression_eta: 0.8459 | Regression_pt: 1.7381 | Regression_sin_phi: 0.4259 | LR: 8.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.9721s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0534 | max_forward: 0.4139 | max_loss: 0.1033 | max_backward: 0.2029 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3771.8 | uss_MB: 3000.0 | child_rss_MB: 4010.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:39:20,708] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:01,569] INFO : Step 11800/50000 rank0 | Train Loss: 9.1380 | Train Loss Unweighted: 3.1130 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7255 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1730 | Regression_eta: 0.0122 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8952 | Classification_binary: 2.7255 | Regression_cos_phi: 0.5840 | Regression_energy: 1.8951 | Regression_eta: 0.6005 | Regression_pt: 2.0148 | Regression_sin_phi: 0.4229 | LR: 8.82e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2084 | max_loss: 0.0031 | max_backward: 0.2861 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.2795 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3771.8 | uss_MB: 3003.2 | child_rss_MB: 4010.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:40:01,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:43,375] INFO : Step 11900/50000 rank0 | Train Loss: 9.0896 | Train Loss Unweighted: 3.1849 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.8375 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1493 | Regression_eta: 0.0172 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9511 | Classification_binary: 2.8375 | Regression_cos_phi: 0.5635 | Regression_energy: 1.6356 | Regression_eta: 0.8438 | Regression_pt: 1.7271 | Regression_sin_phi: 0.5310 | LR: 8.80e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5422s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.2970 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.3735 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3771.8 | uss_MB: 3001.3 | child_rss_MB: 4011.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:40:43,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:24,879] INFO : Step 12000/50000 rank0 | Train Loss: 9.4193 | Train Loss Unweighted: 3.2123 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8391 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1612 | Regression_eta: 0.0186 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9236 | Classification_binary: 2.8391 | Regression_cos_phi: 0.5523 | Regression_energy: 1.7665 | Regression_eta: 0.9149 | Regression_pt: 1.8833 | Regression_sin_phi: 0.5397 | LR: 8.78e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5838s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0030 | max_backward: 0.3080 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.3871 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3771.8 | uss_MB: 3001.4 | child_rss_MB: 4007.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:41:25,093] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:25,112] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 12:41:25,394] INFO : Running validation on rank0 +rank0 [2026-10-28 12:41:25,394] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:41:25,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:41:26,798] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 12:41:26,943] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:41:27,074] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:41:27,148] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:41:27,217] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:41:27,287] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:41:27,356] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:41:27,426] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:41:27,534] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:41:27,665] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:41:27,796] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:41:27,867] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:41:27,937] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:41:28,005] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:41:28,074] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:41:28,143] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:41:28,247] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:41:28,254] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:41:28,257] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:41:28,260] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:41:28,263] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:41:28,264] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:41:28,267] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:41:28,269] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:41:28,270] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:41:28,273] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:41:28,274] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:41:28,277] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:41:28,280] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:41:28,283] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:41:28,285] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:41:28,286] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:41:35,738] INFO : VALIDATION | Step=12000/50000 | Train Loss=9.4193 | Valid Loss=8.8975 | Train Loss Unweighted=3.2123 | Valid Loss Unweighted=3.1692 | Stale=0 | ETA=282.0m +rank0 [2026-10-28 12:41:35,748] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:35,827] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:41:35,830] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:35,983] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:37,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:38,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:39,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:39,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:39,999] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:41:40,003] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:40,094] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:41:40,109] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:40,266] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:42,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:42,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:43,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:43,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:44,057] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:44,060] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:44,137] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:41:44,140] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:41:44,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:46,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:46,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:47,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:48,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:48,154] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:48,162] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:48,425] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:48,827] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:50,723] INFO : Plotted jets +rank0 [2026-10-28 12:41:51,801] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:51,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:52,067] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:52,454] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:53,838] INFO : Plotted jets +rank0 [2026-10-28 12:41:54,945] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:54,978] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:55,185] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:55,547] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:56,920] INFO : Plotted jets +rank0 [2026-10-28 12:41:57,974] INFO : Plotted jet ratio +rank0 [2026-10-28 12:42:39,065] INFO : Step 12100/50000 rank0 | Train Loss: 9.0010 | Train Loss Unweighted: 3.2092 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.8786 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1384 | Regression_eta: 0.0138 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9499 | Classification_binary: 2.8786 | Regression_cos_phi: 0.8022 | Regression_energy: 1.5165 | Regression_eta: 0.6782 | Regression_pt: 1.6584 | Regression_sin_phi: 0.5172 | LR: 8.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5298s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.1859 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1203 | max_tensorboard_logging: 0.3013 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3842.8 | uss_MB: 3074.3 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:42:39,278] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:19,697] INFO : Step 12200/50000 rank0 | Train Loss: 9.3494 | Train Loss Unweighted: 3.2448 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.8844 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1572 | Regression_eta: 0.0212 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9104 | Classification_binary: 2.8844 | Regression_cos_phi: 0.4815 | Regression_energy: 1.7224 | Regression_eta: 1.0425 | Regression_pt: 1.7617 | Regression_sin_phi: 0.5465 | LR: 8.74e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5839s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3040 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 3843.8 | uss_MB: 3075.5 | child_rss_MB: 4009.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:43:19,913] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:01,283] INFO : Step 12300/50000 rank0 | Train Loss: 8.7551 | Train Loss Unweighted: 3.1029 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7495 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1535 | Regression_eta: 0.0103 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0209 | Classification_binary: 2.7495 | Regression_cos_phi: 0.4482 | Regression_energy: 1.6818 | Regression_eta: 0.5062 | Regression_pt: 1.8358 | Regression_sin_phi: 0.5126 | LR: 8.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6897s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1040 | max_loss: 0.0033 | max_backward: 0.3692 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.3812 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3843.8 | uss_MB: 3077.8 | child_rss_MB: 4012.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:44:01,498] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:42,544] INFO : Step 12400/50000 rank0 | Train Loss: 8.9929 | Train Loss Unweighted: 3.2015 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.8457 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1555 | Regression_eta: 0.0137 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0387 | Classification_binary: 2.8457 | Regression_cos_phi: 0.4955 | Regression_energy: 1.7034 | Regression_eta: 0.6756 | Regression_pt: 1.8029 | Regression_sin_phi: 0.4311 | LR: 8.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7646s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0283 | max_forward: 0.1040 | max_loss: 0.0033 | max_backward: 0.3930 | max_optimizer_step: 0.1181 | max_tensorboard_logging: 0.3959 | max_tensorboard_flush: 0.0058 | Memory: rss_MB: 3843.8 | uss_MB: 3077.7 | child_rss_MB: 4015.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:44:42,755] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:22,809] INFO : Step 12500/50000 rank0 | Train Loss: 8.5330 | Train Loss Unweighted: 3.1452 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.7928 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1573 | Regression_eta: 0.0091 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8685 | Classification_binary: 2.7928 | Regression_cos_phi: 0.4293 | Regression_energy: 1.7238 | Regression_eta: 0.4497 | Regression_pt: 1.8525 | Regression_sin_phi: 0.4164 | LR: 8.68e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5909s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2757 | max_optimizer_step: 0.1202 | max_tensorboard_logging: 0.3897 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3843.8 | uss_MB: 3076.0 | child_rss_MB: 4014.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:45:23,023] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:03,291] INFO : Step 12600/50000 rank0 | Train Loss: 8.6002 | Train Loss Unweighted: 3.1142 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7823 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1426 | Regression_eta: 0.0113 | Regression_pt: 0.1411 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0279 | Classification_binary: 2.7823 | Regression_cos_phi: 0.5066 | Regression_energy: 1.5628 | Regression_eta: 0.5557 | Regression_pt: 1.6927 | Regression_sin_phi: 0.4724 | LR: 8.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5545s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0612 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1586 | max_tensorboard_logging: 0.3210 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 3843.8 | uss_MB: 3076.0 | child_rss_MB: 4015.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:46:03,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:45,480] INFO : Step 12700/50000 rank0 | Train Loss: 8.5151 | Train Loss Unweighted: 3.1102 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7829 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1427 | Regression_eta: 0.0124 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0210 | Classification_binary: 2.7829 | Regression_cos_phi: 0.4765 | Regression_energy: 1.5634 | Regression_eta: 0.6105 | Regression_pt: 1.6397 | Regression_sin_phi: 0.4210 | LR: 8.64e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6739s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.3093 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.4066 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3843.1 | uss_MB: 3074.3 | child_rss_MB: 4015.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:46:45,693] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:26,561] INFO : Step 12800/50000 rank0 | Train Loss: 8.8267 | Train Loss Unweighted: 3.1154 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7680 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1532 | Regression_eta: 0.0112 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9373 | Classification_binary: 2.7680 | Regression_cos_phi: 0.6415 | Regression_energy: 1.6791 | Regression_eta: 0.5525 | Regression_pt: 1.7472 | Regression_sin_phi: 0.5012 | LR: 8.62e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4890s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0377 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.2026 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3845.1 | uss_MB: 3075.5 | child_rss_MB: 4017.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:47:26,791] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:08,048] INFO : Step 12900/50000 rank0 | Train Loss: 8.8230 | Train Loss Unweighted: 3.1213 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7778 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1483 | Regression_eta: 0.0138 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9264 | Classification_binary: 2.7778 | Regression_cos_phi: 0.5238 | Regression_energy: 1.6244 | Regression_eta: 0.6770 | Regression_pt: 1.7439 | Regression_sin_phi: 0.5496 | LR: 8.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5131s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0503 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.2033 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0143 | Memory: rss_MB: 3845.1 | uss_MB: 3075.6 | child_rss_MB: 4015.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:48:08,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:49,543] INFO : Step 13000/50000 rank0 | Train Loss: 8.9021 | Train Loss Unweighted: 3.1906 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.8431 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1511 | Regression_eta: 0.0117 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9532 | Classification_binary: 2.8431 | Regression_cos_phi: 0.4663 | Regression_energy: 1.6559 | Regression_eta: 0.5772 | Regression_pt: 1.7676 | Regression_sin_phi: 0.6388 | LR: 8.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0800 | max_backward: 0.2875 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.3044 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3845.1 | uss_MB: 3077.8 | child_rss_MB: 4018.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:48:49,756] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:49,775] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 12:48:49,951] INFO : Running validation on rank0 +rank0 [2026-10-28 12:48:49,951] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:48:49,951] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:48:51,265] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 12:48:51,411] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:48:51,542] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:48:51,617] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:48:51,686] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:48:51,754] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:48:51,823] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:48:51,892] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:48:52,002] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:48:52,133] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:48:52,264] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:48:52,334] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:48:52,403] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:48:52,471] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:48:52,540] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:48:52,610] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:48:52,732] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:48:52,739] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:48:52,742] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:48:52,745] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:48:52,746] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:48:52,749] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:48:52,751] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:48:52,752] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:48:52,755] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:48:52,758] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:48:52,759] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:48:52,762] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:48:52,765] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:48:52,766] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:48:52,768] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:48:52,769] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:49:00,227] INFO : VALIDATION | Step=13000/50000 | Train Loss=8.9021 | Valid Loss=8.8012 | Train Loss Unweighted=3.1906 | Valid Loss Unweighted=3.1547 | Stale=0 | ETA=274.5m +rank0 [2026-10-28 12:49:00,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:00,299] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:49:00,303] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:49:00,456] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:02,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:03,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:03,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:04,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:04,535] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:49:04,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:04,597] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:49:04,600] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:04,683] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:06,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:07,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:07,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:08,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:08,667] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:49:08,670] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:08,726] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:49:08,729] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:49:08,819] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:10,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:11,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:11,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:12,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:12,702] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:49:12,709] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:49:12,964] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:49:13,370] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:15,299] INFO : Plotted jets +rank0 [2026-10-28 12:49:16,377] INFO : Plotted jet ratio +rank0 [2026-10-28 12:49:16,411] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:16,638] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:17,024] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:18,410] INFO : Plotted jets +rank0 [2026-10-28 12:49:19,513] INFO : Plotted jet ratio +rank0 [2026-10-28 12:49:19,571] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 12:49:19,793] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 12:49:20,154] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:21,529] INFO : Plotted jets +rank0 [2026-10-28 12:49:22,579] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:04,359] INFO : Step 13100/50000 rank0 | Train Loss: 9.1622 | Train Loss Unweighted: 3.1034 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.7317 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1631 | Regression_eta: 0.0142 | Regression_pt: 0.1569 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0181 | Classification_binary: 2.7317 | Regression_cos_phi: 0.5608 | Regression_energy: 1.7868 | Regression_eta: 0.6958 | Regression_pt: 1.8817 | Regression_sin_phi: 0.4875 | LR: 8.56e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5991s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2469 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3901 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4051.5 | uss_MB: 3283.1 | child_rss_MB: 4012.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:50:04,572] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:46,631] INFO : Step 13200/50000 rank0 | Train Loss: 10.1645 | Train Loss Unweighted: 3.2709 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8538 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1850 | Regression_eta: 0.0216 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9236 | Classification_binary: 2.8538 | Regression_cos_phi: 0.6024 | Regression_energy: 2.0268 | Regression_eta: 1.0640 | Regression_pt: 2.0621 | Regression_sin_phi: 0.6318 | LR: 8.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0655 | max_backward: 0.2028 | max_optimizer_step: 0.1194 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3918.0 | uss_MB: 3149.4 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:50:46,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:27,809] INFO : Step 13300/50000 rank0 | Train Loss: 8.6329 | Train Loss Unweighted: 3.0879 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7406 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1522 | Regression_eta: 0.0117 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9176 | Classification_binary: 2.7406 | Regression_cos_phi: 0.4419 | Regression_energy: 1.6671 | Regression_eta: 0.5772 | Regression_pt: 1.7925 | Regression_sin_phi: 0.4959 | LR: 8.52e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.7605s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.2053 | max_loss: 0.0032 | max_backward: 0.2868 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.3091 | max_tensorboard_flush: 0.0132 | Memory: rss_MB: 3918.0 | uss_MB: 3152.1 | child_rss_MB: 4012.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:51:28,025] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:08,453] INFO : Step 13400/50000 rank0 | Train Loss: 8.5197 | Train Loss Unweighted: 3.2077 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.8697 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1496 | Regression_eta: 0.0101 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9050 | Classification_binary: 2.8697 | Regression_cos_phi: 0.4663 | Regression_energy: 1.6388 | Regression_eta: 0.4968 | Regression_pt: 1.7495 | Regression_sin_phi: 0.3935 | LR: 8.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5417s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0158 | max_backward: 0.2027 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.3981 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3918.0 | uss_MB: 3150.1 | child_rss_MB: 4012.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:52:08,668] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:48,696] INFO : Step 13500/50000 rank0 | Train Loss: 9.5619 | Train Loss Unweighted: 3.2416 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8633 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1655 | Regression_eta: 0.0152 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9946 | Classification_binary: 2.8633 | Regression_cos_phi: 0.6292 | Regression_energy: 1.8136 | Regression_eta: 0.7477 | Regression_pt: 1.8869 | Regression_sin_phi: 0.6265 | LR: 8.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5339s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1851 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1180 | max_tensorboard_logging: 0.2984 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3918.0 | uss_MB: 3150.1 | child_rss_MB: 4010.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:52:48,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:29,364] INFO : Step 13600/50000 rank0 | Train Loss: 8.0709 | Train Loss Unweighted: 3.0367 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7102 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1451 | Regression_eta: 0.0097 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8772 | Classification_binary: 2.7102 | Regression_cos_phi: 0.4050 | Regression_energy: 1.5899 | Regression_eta: 0.4780 | Regression_pt: 1.7056 | Regression_sin_phi: 0.3051 | LR: 8.45e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8253s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0605 | max_forward: 0.1038 | max_loss: 0.0030 | max_backward: 0.3944 | max_optimizer_step: 0.1199 | max_tensorboard_logging: 0.3984 | max_tensorboard_flush: 0.0145 | Memory: rss_MB: 3918.0 | uss_MB: 3152.1 | child_rss_MB: 4011.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:53:29,577] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:10,640] INFO : Step 13700/50000 rank0 | Train Loss: 8.2451 | Train Loss Unweighted: 3.0347 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.7217 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1352 | Regression_eta: 0.0104 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9106 | Classification_binary: 2.7217 | Regression_cos_phi: 0.5739 | Regression_energy: 1.4809 | Regression_eta: 0.5110 | Regression_pt: 1.5873 | Regression_sin_phi: 0.4597 | LR: 8.43e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7538s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1043 | max_loss: 0.0031 | max_backward: 0.3944 | max_optimizer_step: 0.1196 | max_tensorboard_logging: 0.3999 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 3918.0 | uss_MB: 3152.2 | child_rss_MB: 4012.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 12:54:10,854] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:51,013] INFO : Step 13800/50000 rank0 | Train Loss: 8.9969 | Train Loss Unweighted: 3.1875 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.8209 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1621 | Regression_eta: 0.0117 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0126 | Classification_binary: 2.8209 | Regression_cos_phi: 0.4949 | Regression_energy: 1.7764 | Regression_eta: 0.5769 | Regression_pt: 1.8832 | Regression_sin_phi: 0.4320 | LR: 8.41e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4986s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0449 | max_forward: 0.0772 | max_loss: 0.0030 | max_backward: 0.2020 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3918.0 | uss_MB: 3149.5 | child_rss_MB: 4008.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:54:51,240] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:32,142] INFO : Step 13900/50000 rank0 | Train Loss: 8.9735 | Train Loss Unweighted: 3.1339 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7748 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1596 | Regression_eta: 0.0139 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9382 | Classification_binary: 2.7748 | Regression_cos_phi: 0.5937 | Regression_energy: 1.7485 | Regression_eta: 0.6811 | Regression_pt: 1.8012 | Regression_sin_phi: 0.4360 | LR: 8.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6073s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0030 | max_backward: 0.3061 | max_optimizer_step: 0.1180 | max_tensorboard_logging: 0.3870 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3918.0 | uss_MB: 3149.4 | child_rss_MB: 4010.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:55:32,354] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:13,305] INFO : Step 14000/50000 rank0 | Train Loss: 8.5783 | Train Loss Unweighted: 2.9954 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6524 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1498 | Regression_eta: 0.0116 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9289 | Classification_binary: 2.6524 | Regression_cos_phi: 0.4507 | Regression_energy: 1.6415 | Regression_eta: 0.5726 | Regression_pt: 1.7495 | Regression_sin_phi: 0.5827 | LR: 8.37e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6537s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2087 | max_loss: 0.0032 | max_backward: 0.2863 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3919.0 | uss_MB: 3152.1 | child_rss_MB: 4011.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:56:13,535] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:13,555] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 12:56:13,738] INFO : Running validation on rank0 +rank0 [2026-10-28 12:56:13,738] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:56:13,738] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:56:15,102] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 12:56:15,248] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:56:15,383] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:56:15,458] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:56:15,529] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:56:15,599] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:56:15,669] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:56:15,739] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:56:15,848] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:56:15,979] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:56:16,112] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:56:16,183] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:56:16,254] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:56:16,324] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:56:16,394] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:56:16,464] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:56:16,565] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:56:16,571] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:56:16,573] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:56:16,576] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:56:16,577] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:56:16,581] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:56:16,583] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:56:16,586] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:56:16,587] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:56:16,590] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:56:16,591] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:56:16,593] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:56:16,596] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:56:16,600] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:56:16,602] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:56:16,605] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:56:24,351] INFO : VALIDATION | Step=14000/50000 | Train Loss=8.5783 | Valid Loss=8.7057 | Train Loss Unweighted=2.9954 | Valid Loss Unweighted=3.1052 | Stale=0 | ETA=267.0m +rank0 [2026-10-28 12:56:24,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:24,483] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:56:24,487] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:56:24,682] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:26,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:27,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:27,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:28,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:28,793] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:56:28,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:28,868] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:56:28,872] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:28,947] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:30,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:31,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:32,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:32,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:32,861] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:56:32,864] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:32,918] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:56:32,921] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:56:33,026] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:34,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:35,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:36,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:36,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:36,974] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:56:36,981] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:56:37,238] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:56:37,643] INFO : Plotted number of elements +rank0 [2026-10-28 12:56:39,578] INFO : Plotted jets +rank0 [2026-10-28 12:56:40,660] INFO : Plotted jet ratio +rank0 [2026-10-28 12:56:40,691] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:40,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:41,311] INFO : Plotted number of elements +rank0 [2026-10-28 12:56:42,711] INFO : Plotted jets +rank0 [2026-10-28 12:56:43,818] INFO : Plotted jet ratio +rank0 [2026-10-28 12:56:43,850] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 12:56:44,058] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 12:56:44,421] INFO : Plotted number of elements +rank0 [2026-10-28 12:56:45,801] INFO : Plotted jets +rank0 [2026-10-28 12:56:46,864] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:27,990] INFO : Step 14100/50000 rank0 | Train Loss: 8.3070 | Train Loss Unweighted: 3.0408 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.7105 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1455 | Regression_eta: 0.0107 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9159 | Classification_binary: 2.7105 | Regression_cos_phi: 0.4309 | Regression_energy: 1.5938 | Regression_eta: 0.5281 | Regression_pt: 1.6964 | Regression_sin_phi: 0.4314 | LR: 8.35e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6098s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2521 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.3945 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4136.1 | uss_MB: 3369.9 | child_rss_MB: 4008.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:57:28,203] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:08,699] INFO : Step 14200/50000 rank0 | Train Loss: 9.0817 | Train Loss Unweighted: 3.0015 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6615 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1449 | Regression_eta: 0.0246 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8920 | Classification_binary: 2.6615 | Regression_cos_phi: 0.6007 | Regression_energy: 1.5877 | Regression_eta: 1.2069 | Regression_pt: 1.6134 | Regression_sin_phi: 0.5196 | LR: 8.32e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5684s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2955 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.3723 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3990.1 | uss_MB: 3223.5 | child_rss_MB: 4012.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:58:08,914] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:48,952] INFO : Step 14300/50000 rank0 | Train Loss: 8.0664 | Train Loss Unweighted: 3.0189 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6956 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1429 | Regression_eta: 0.0088 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9264 | Classification_binary: 2.6956 | Regression_cos_phi: 0.4059 | Regression_energy: 1.5662 | Regression_eta: 0.4337 | Regression_pt: 1.6808 | Regression_sin_phi: 0.3579 | LR: 8.30e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4163s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0934 | max_loss: 0.0036 | max_backward: 0.2034 | max_optimizer_step: 0.1171 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3990.1 | uss_MB: 3223.8 | child_rss_MB: 4011.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:58:49,166] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:29,715] INFO : Step 14400/50000 rank0 | Train Loss: 9.2970 | Train Loss Unweighted: 3.0879 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.7161 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1644 | Regression_eta: 0.0204 | Regression_pt: 0.1512 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9820 | Classification_binary: 2.7161 | Regression_cos_phi: 0.5113 | Regression_energy: 1.8010 | Regression_eta: 1.0024 | Regression_pt: 1.8127 | Regression_sin_phi: 0.4714 | LR: 8.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5067s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1844 | max_loss: 0.0031 | max_backward: 0.2025 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3990.1 | uss_MB: 3223.8 | child_rss_MB: 4013.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:59:29,931] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:10,238] INFO : Step 14500/50000 rank0 | Train Loss: 8.7149 | Train Loss Unweighted: 3.1416 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.7848 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1561 | Regression_eta: 0.0117 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9309 | Classification_binary: 2.7848 | Regression_cos_phi: 0.4020 | Regression_energy: 1.7102 | Regression_eta: 0.5745 | Regression_pt: 1.8736 | Regression_sin_phi: 0.4389 | LR: 8.26e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6171s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0032 | max_backward: 0.3110 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3990.1 | uss_MB: 3223.8 | child_rss_MB: 4008.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:00:10,450] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:50,640] INFO : Step 14600/50000 rank0 | Train Loss: 8.4265 | Train Loss Unweighted: 3.1238 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.7807 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1536 | Regression_eta: 0.0105 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8914 | Classification_binary: 2.7807 | Regression_cos_phi: 0.3978 | Regression_energy: 1.6833 | Regression_eta: 0.5151 | Regression_pt: 1.7758 | Regression_sin_phi: 0.3823 | LR: 8.24e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5110s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1741 | max_loss: 0.0032 | max_backward: 0.2025 | max_optimizer_step: 0.1182 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3990.1 | uss_MB: 3223.9 | child_rss_MB: 4009.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:00:50,858] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:31,979] INFO : Step 14700/50000 rank0 | Train Loss: 8.8976 | Train Loss Unweighted: 3.1662 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.8005 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1615 | Regression_eta: 0.0109 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8995 | Classification_binary: 2.8005 | Regression_cos_phi: 0.5254 | Regression_energy: 1.7701 | Regression_eta: 0.5341 | Regression_pt: 1.9078 | Regression_sin_phi: 0.4603 | LR: 8.21e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5925s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1047 | max_loss: 0.0531 | max_backward: 0.2862 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.2923 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3990.1 | uss_MB: 3226.5 | child_rss_MB: 4015.2 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 13:01:32,192] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:13,809] INFO : Step 14800/50000 rank0 | Train Loss: 8.6269 | Train Loss Unweighted: 3.1356 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8058 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1422 | Regression_eta: 0.0128 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9223 | Classification_binary: 2.8058 | Regression_cos_phi: 0.5211 | Regression_energy: 1.5579 | Regression_eta: 0.6283 | Regression_pt: 1.6712 | Regression_sin_phi: 0.5203 | LR: 8.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1533 | max_loss: 0.0034 | max_backward: 0.2863 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3990.5 | uss_MB: 3225.8 | child_rss_MB: 4013.4 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 13:02:14,038] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:54,497] INFO : Step 14900/50000 rank0 | Train Loss: 9.2290 | Train Loss Unweighted: 3.2223 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8366 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1710 | Regression_eta: 0.0108 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9667 | Classification_binary: 2.8366 | Regression_cos_phi: 0.5257 | Regression_energy: 1.8741 | Regression_eta: 0.5286 | Regression_pt: 2.0133 | Regression_sin_phi: 0.4841 | LR: 8.17e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5177s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3990.5 | uss_MB: 3224.0 | child_rss_MB: 4011.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:02:54,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:35,766] INFO : Step 15000/50000 rank0 | Train Loss: 9.0036 | Train Loss Unweighted: 3.0300 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6785 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1491 | Regression_eta: 0.0178 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0021 | Classification_binary: 2.6785 | Regression_cos_phi: 0.5335 | Regression_energy: 1.6339 | Regression_eta: 0.8768 | Regression_pt: 1.7663 | Regression_sin_phi: 0.5126 | LR: 8.15e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6193s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3108 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3990.5 | uss_MB: 3224.0 | child_rss_MB: 4008.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:03:35,980] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:35,999] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 13:03:36,275] INFO : Running validation on rank0 +rank0 [2026-10-28 13:03:36,275] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:03:36,275] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:03:37,664] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 13:03:37,809] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:03:37,940] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:03:38,015] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:03:38,083] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:03:38,152] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:03:38,221] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:03:38,290] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:03:38,400] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:03:38,532] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:03:38,664] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:03:38,735] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:03:38,805] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:03:38,875] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:03:38,946] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:03:39,017] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:03:39,127] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:03:39,134] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:03:39,136] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:03:39,139] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:03:39,140] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:03:39,143] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:03:39,146] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:03:39,148] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:03:39,150] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:03:39,154] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:03:39,154] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:03:39,157] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:03:39,160] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:03:39,163] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:03:39,166] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:03:39,167] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:03:46,614] INFO : VALIDATION | Step=15000/50000 | Train Loss=9.0036 | Valid Loss=8.7900 | Train Loss Unweighted=3.0300 | Valid Loss Unweighted=3.1191 | Stale=1 | ETA=259.5m +rank0 [2026-10-28 13:03:46,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:46,739] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:03:46,742] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:03:46,946] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:48,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:49,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:50,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:50,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:50,973] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:03:50,976] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:51,054] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:03:51,057] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:03:51,124] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:53,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:53,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:54,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:54,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:55,006] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:03:55,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:55,081] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:03:55,084] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:03:55,177] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:56,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:57,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:58,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:59,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:59,125] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:03:59,133] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:03:59,394] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:03:59,808] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:01,245] INFO : Plotted jets +rank0 [2026-10-28 13:04:02,835] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:02,867] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:03,102] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:03,490] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:04,879] INFO : Plotted jets +rank0 [2026-10-28 13:04:05,985] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:06,017] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 13:04:06,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 13:04:06,584] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:07,958] INFO : Plotted jets +rank0 [2026-10-28 13:04:09,019] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:51,101] INFO : Step 15100/50000 rank0 | Train Loss: 8.3137 | Train Loss Unweighted: 3.1305 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7994 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1469 | Regression_eta: 0.0104 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9362 | Classification_binary: 2.7994 | Regression_cos_phi: 0.4070 | Regression_energy: 1.6091 | Regression_eta: 0.5113 | Regression_pt: 1.7082 | Regression_sin_phi: 0.3425 | LR: 8.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1099 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4196.1 | uss_MB: 3428.5 | child_rss_MB: 4008.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:04:51,326] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:32,321] INFO : Step 15200/50000 rank0 | Train Loss: 8.6116 | Train Loss Unweighted: 3.0664 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.7244 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1497 | Regression_eta: 0.0132 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9641 | Classification_binary: 2.7244 | Regression_cos_phi: 0.4789 | Regression_energy: 1.6406 | Regression_eta: 0.6475 | Regression_pt: 1.7366 | Regression_sin_phi: 0.4194 | LR: 8.10e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5302s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1788 | max_loss: 0.0033 | max_backward: 0.2034 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.2901 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4008.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:05:32,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:14,491] INFO : Step 15300/50000 rank0 | Train Loss: 8.1949 | Train Loss Unweighted: 3.0739 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7564 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1383 | Regression_eta: 0.0120 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.7564 | Regression_cos_phi: 0.4643 | Regression_energy: 1.5159 | Regression_eta: 0.5910 | Regression_pt: 1.6314 | Regression_sin_phi: 0.3758 | LR: 8.08e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5454s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0567 | max_backward: 0.2035 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.3613 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:06:14,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:55,926] INFO : Step 15400/50000 rank0 | Train Loss: 9.5427 | Train Loss Unweighted: 3.2858 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8852 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1795 | Regression_eta: 0.0149 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8925 | Classification_binary: 2.8852 | Regression_cos_phi: 0.5740 | Regression_energy: 1.9672 | Regression_eta: 0.7305 | Regression_pt: 2.0631 | Regression_sin_phi: 0.4303 | LR: 8.05e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3071 | max_optimizer_step: 0.1218 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4010.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:06:56,141] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:37,472] INFO : Step 15500/50000 rank0 | Train Loss: 8.2147 | Train Loss Unweighted: 3.0525 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.7418 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1345 | Regression_eta: 0.0120 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9323 | Classification_binary: 2.7418 | Regression_cos_phi: 0.4910 | Regression_energy: 1.4734 | Regression_eta: 0.5877 | Regression_pt: 1.5644 | Regression_sin_phi: 0.4241 | LR: 8.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5143s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1823 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4009.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:07:37,684] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:19,214] INFO : Step 15600/50000 rank0 | Train Loss: 8.5969 | Train Loss Unweighted: 3.0652 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7274 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1479 | Regression_eta: 0.0110 | Regression_pt: 0.1433 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8948 | Classification_binary: 2.7274 | Regression_cos_phi: 0.6436 | Regression_energy: 1.6207 | Regression_eta: 0.5431 | Regression_pt: 1.7181 | Regression_sin_phi: 0.4494 | LR: 8.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7215s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2113 | max_loss: 0.0032 | max_backward: 0.2862 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.2788 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4065.3 | uss_MB: 3301.1 | child_rss_MB: 4009.8 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 96.9 +rank0 [2026-10-28 13:08:19,438] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:00,426] INFO : Step 15700/50000 rank0 | Train Loss: 8.8387 | Train Loss Unweighted: 3.0443 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6841 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1609 | Regression_eta: 0.0154 | Regression_pt: 0.1508 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8584 | Classification_binary: 2.6841 | Regression_cos_phi: 0.4512 | Regression_energy: 1.7635 | Regression_eta: 0.7590 | Regression_pt: 1.8089 | Regression_sin_phi: 0.5136 | LR: 7.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5383s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.2275 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4066.3 | uss_MB: 3299.1 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:09:00,639] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:40,910] INFO : Step 15800/50000 rank0 | Train Loss: 8.5959 | Train Loss Unweighted: 3.1063 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7684 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1510 | Regression_eta: 0.0124 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9291 | Classification_binary: 2.7684 | Regression_cos_phi: 0.5266 | Regression_energy: 1.6549 | Regression_eta: 0.6080 | Regression_pt: 1.6819 | Regression_sin_phi: 0.4271 | LR: 7.96e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5871s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2933 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.3873 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4066.4 | uss_MB: 3298.8 | child_rss_MB: 4009.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 13:09:41,121] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:22,713] INFO : Step 15900/50000 rank0 | Train Loss: 8.5419 | Train Loss Unweighted: 3.0809 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.7518 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1418 | Regression_eta: 0.0124 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9083 | Classification_binary: 2.7518 | Regression_cos_phi: 0.5806 | Regression_energy: 1.5541 | Regression_eta: 0.6085 | Regression_pt: 1.6755 | Regression_sin_phi: 0.4630 | LR: 7.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6046s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1666 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4066.4 | uss_MB: 3299.0 | child_rss_MB: 4010.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:10:22,944] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:04,181] INFO : Step 16000/50000 rank0 | Train Loss: 8.0822 | Train Loss Unweighted: 2.8681 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5479 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1396 | Regression_eta: 0.0121 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9008 | Classification_binary: 2.5479 | Regression_cos_phi: 0.5154 | Regression_energy: 1.5292 | Regression_eta: 0.5929 | Regression_pt: 1.6316 | Regression_sin_phi: 0.3645 | LR: 7.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5273s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0783 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.2821 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4066.4 | uss_MB: 3299.0 | child_rss_MB: 4011.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:11:04,410] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:04,429] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 13:11:04,600] INFO : Running validation on rank0 +rank0 [2026-10-28 13:11:04,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:11:04,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:11:05,925] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 13:11:06,070] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:11:06,201] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:11:06,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:11:06,345] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:11:06,414] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:11:06,483] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:11:06,552] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:11:06,661] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:11:06,791] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:11:06,923] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:11:06,995] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:11:07,064] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:11:07,135] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:11:07,204] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:11:07,273] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:11:07,387] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:11:07,394] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:11:07,397] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:11:07,400] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:11:07,401] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:11:07,404] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:11:07,407] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:11:07,408] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:11:07,411] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:11:07,413] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:11:07,414] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:11:07,417] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:11:07,418] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:11:07,421] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:11:07,424] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:11:07,426] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:11:15,075] INFO : VALIDATION | Step=16000/50000 | Train Loss=8.0822 | Valid Loss=8.6015 | Train Loss Unweighted=2.8681 | Valid Loss Unweighted=3.0532 | Stale=0 | ETA=252.2m +rank0 [2026-10-28 13:11:15,084] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:15,142] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:11:15,146] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:11:15,301] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:17,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:17,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:18,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:19,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:19,439] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:11:19,442] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:19,498] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:11:19,500] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:19,592] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:21,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:22,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:22,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:23,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:23,678] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:11:23,681] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:23,736] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:11:23,739] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:11:23,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:25,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:26,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:26,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:27,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:27,753] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:11:27,760] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:11:28,017] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:11:28,422] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:29,851] INFO : Plotted jets +rank0 [2026-10-28 13:11:31,426] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:31,458] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:31,684] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:32,073] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:33,490] INFO : Plotted jets +rank0 [2026-10-28 13:11:34,595] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:34,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 13:11:34,836] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 13:11:35,197] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:36,575] INFO : Plotted jets +rank0 [2026-10-28 13:11:37,619] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:19,199] INFO : Step 16100/50000 rank0 | Train Loss: 9.0595 | Train Loss Unweighted: 3.1293 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7254 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1814 | Regression_eta: 0.0079 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9610 | Classification_binary: 2.7254 | Regression_cos_phi: 0.4355 | Regression_energy: 1.9879 | Regression_eta: 0.3905 | Regression_pt: 2.1780 | Regression_sin_phi: 0.3812 | LR: 7.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6175s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2693 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4273.4 | uss_MB: 3507.1 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:12:19,412] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:59,642] INFO : Step 16200/50000 rank0 | Train Loss: 8.2823 | Train Loss Unweighted: 3.0601 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7381 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1412 | Regression_eta: 0.0096 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9543 | Classification_binary: 2.7381 | Regression_cos_phi: 0.5232 | Regression_energy: 1.5469 | Regression_eta: 0.4712 | Regression_pt: 1.6407 | Regression_sin_phi: 0.4079 | LR: 7.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4910s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0423 | max_forward: 0.0781 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1632 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4139.2 | uss_MB: 3371.5 | child_rss_MB: 4006.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:12:59,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:40,649] INFO : Step 16300/50000 rank0 | Train Loss: 8.4116 | Train Loss Unweighted: 3.0740 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.7465 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1434 | Regression_eta: 0.0108 | Regression_pt: 0.1388 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9234 | Classification_binary: 2.7465 | Regression_cos_phi: 0.5246 | Regression_energy: 1.5707 | Regression_eta: 0.5297 | Regression_pt: 1.6645 | Regression_sin_phi: 0.4521 | LR: 7.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5812s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1221 | max_loss: 0.0033 | max_backward: 0.2033 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4139.2 | uss_MB: 3371.6 | child_rss_MB: 4005.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:13:40,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:22,738] INFO : Step 16400/50000 rank0 | Train Loss: 8.6753 | Train Loss Unweighted: 3.0365 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6754 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1609 | Regression_eta: 0.0131 | Regression_pt: 0.1549 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9421 | Classification_binary: 2.6754 | Regression_cos_phi: 0.4213 | Regression_energy: 1.7631 | Regression_eta: 0.6421 | Regression_pt: 1.8574 | Regression_sin_phi: 0.3737 | LR: 7.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6423s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0033 | max_backward: 0.3097 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3848 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4139.2 | uss_MB: 3371.7 | child_rss_MB: 4010.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:14:22,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:04,345] INFO : Step 16500/50000 rank0 | Train Loss: 8.9450 | Train Loss Unweighted: 2.9719 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5974 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1652 | Regression_eta: 0.0127 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9362 | Classification_binary: 2.5974 | Regression_cos_phi: 0.5817 | Regression_energy: 1.8104 | Regression_eta: 0.6226 | Regression_pt: 1.9277 | Regression_sin_phi: 0.4690 | LR: 7.79e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.6207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.3107 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4139.2 | uss_MB: 3371.8 | child_rss_MB: 4010.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:15:04,558] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:44,553] INFO : Step 16600/50000 rank0 | Train Loss: 8.7031 | Train Loss Unweighted: 3.0259 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6836 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1473 | Regression_eta: 0.0129 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9708 | Classification_binary: 2.6836 | Regression_cos_phi: 0.5514 | Regression_energy: 1.6138 | Regression_eta: 0.6331 | Regression_pt: 1.7435 | Regression_sin_phi: 0.5068 | LR: 7.77e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0034 | max_backward: 0.3112 | max_optimizer_step: 0.1228 | max_tensorboard_logging: 0.3847 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4139.2 | uss_MB: 3371.8 | child_rss_MB: 4009.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:15:44,764] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:27,081] INFO : Step 16700/50000 rank0 | Train Loss: 9.3254 | Train Loss Unweighted: 3.1841 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7857 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1774 | Regression_eta: 0.0119 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9624 | Classification_binary: 2.7857 | Regression_cos_phi: 0.4460 | Regression_energy: 1.9436 | Regression_eta: 0.5829 | Regression_pt: 2.0843 | Regression_sin_phi: 0.5204 | LR: 7.74e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5367s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0033 | max_backward: 0.2272 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4139.2 | uss_MB: 3371.9 | child_rss_MB: 4010.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:16:27,293] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:08,044] INFO : Step 16800/50000 rank0 | Train Loss: 9.0427 | Train Loss Unweighted: 3.0949 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7374 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1551 | Regression_eta: 0.0167 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9031 | Classification_binary: 2.7374 | Regression_cos_phi: 0.5770 | Regression_energy: 1.6995 | Regression_eta: 0.8215 | Regression_pt: 1.7995 | Regression_sin_phi: 0.5046 | LR: 7.72e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0989 | max_loss: 0.0033 | max_backward: 0.2029 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4138.4 | uss_MB: 3371.6 | child_rss_MB: 4008.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:17:08,257] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:50,670] INFO : Step 16900/50000 rank0 | Train Loss: 8.6535 | Train Loss Unweighted: 3.0307 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7227 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1276 | Regression_eta: 0.0206 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9626 | Classification_binary: 2.7227 | Regression_cos_phi: 0.5511 | Regression_energy: 1.3986 | Regression_eta: 1.0148 | Regression_pt: 1.4714 | Regression_sin_phi: 0.5324 | LR: 7.69e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8296s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1068 | max_forward: 0.1048 | max_loss: 0.0033 | max_backward: 0.3952 | max_optimizer_step: 0.1207 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4138.4 | uss_MB: 3373.9 | child_rss_MB: 4010.0 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 13:17:50,882] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:32,305] INFO : Step 17000/50000 rank0 | Train Loss: 8.7193 | Train Loss Unweighted: 3.0443 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6958 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1533 | Regression_eta: 0.0168 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9446 | Classification_binary: 2.6958 | Regression_cos_phi: 0.4264 | Regression_energy: 1.6795 | Regression_eta: 0.8258 | Regression_pt: 1.7466 | Regression_sin_phi: 0.4006 | LR: 7.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6300s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0786 | max_loss: 0.0034 | max_backward: 0.3086 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.3902 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4138.4 | uss_MB: 3372.0 | child_rss_MB: 4011.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:18:32,519] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:32,538] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 13:18:32,817] INFO : Running validation on rank0 +rank0 [2026-10-28 13:18:32,817] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:18:32,817] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:18:34,218] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 13:18:34,363] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:18:34,497] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:18:34,571] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:18:34,640] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:18:34,710] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:18:34,779] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:18:34,850] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:18:34,960] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:18:35,093] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:18:35,224] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:18:35,296] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:18:35,365] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:18:35,433] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:18:35,503] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:18:35,574] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:18:35,684] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:18:35,691] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:18:35,694] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:18:35,697] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:18:35,700] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:18:35,701] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:18:35,704] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:18:35,707] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:18:35,708] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:18:35,710] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:18:35,711] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:18:35,714] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:18:35,717] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:18:35,718] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:18:35,720] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:18:35,723] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:18:43,213] INFO : VALIDATION | Step=17000/50000 | Train Loss=8.7193 | Valid Loss=8.4506 | Train Loss Unweighted=3.0443 | Valid Loss Unweighted=3.0090 | Stale=0 | ETA=244.9m +rank0 [2026-10-28 13:18:43,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:43,322] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:18:43,326] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:18:43,481] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:45,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:46,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:46,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:47,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:47,535] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:18:47,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:47,606] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:18:47,609] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:47,689] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:49,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:50,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:50,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:51,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:51,681] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:18:51,684] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:51,753] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:18:51,756] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:18:51,837] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:53,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:54,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:54,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:55,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:55,711] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:18:55,719] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:18:55,977] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:18:56,382] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:58,320] INFO : Plotted jets +rank0 [2026-10-28 13:18:59,405] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:59,439] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:59,668] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:00,059] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:01,450] INFO : Plotted jets +rank0 [2026-10-28 13:19:02,556] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:02,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:02,795] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:03,160] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:04,548] INFO : Plotted jets +rank0 [2026-10-28 13:19:05,603] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:47,105] INFO : Step 17100/50000 rank0 | Train Loss: 8.3476 | Train Loss Unweighted: 2.9992 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6652 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1475 | Regression_eta: 0.0114 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9343 | Classification_binary: 2.6652 | Regression_cos_phi: 0.4370 | Regression_energy: 1.6160 | Regression_eta: 0.5612 | Regression_pt: 1.7018 | Regression_sin_phi: 0.4320 | LR: 7.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6067s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1441 | max_loss: 0.0034 | max_backward: 0.2025 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.2922 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4348.4 | uss_MB: 3583.2 | child_rss_MB: 4008.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:19:47,332] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:28,999] INFO : Step 17200/50000 rank0 | Train Loss: 8.0936 | Train Loss Unweighted: 2.9793 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6702 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1346 | Regression_eta: 0.0114 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9661 | Classification_binary: 2.6702 | Regression_cos_phi: 0.4725 | Regression_energy: 1.4743 | Regression_eta: 0.5599 | Regression_pt: 1.5498 | Regression_sin_phi: 0.4008 | LR: 7.62e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3105 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.3706 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4210.6 | uss_MB: 3444.7 | child_rss_MB: 4007.4 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:20:29,212] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:09,537] INFO : Step 17300/50000 rank0 | Train Loss: 8.4364 | Train Loss Unweighted: 3.0296 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6768 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1575 | Regression_eta: 0.0115 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.6768 | Regression_cos_phi: 0.4129 | Regression_energy: 1.7259 | Regression_eta: 0.5662 | Regression_pt: 1.8441 | Regression_sin_phi: 0.3504 | LR: 7.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5243s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0510 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.2940 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4210.6 | uss_MB: 3444.9 | child_rss_MB: 4005.7 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:21:09,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:50,337] INFO : Step 17400/50000 rank0 | Train Loss: 8.6185 | Train Loss Unweighted: 3.0284 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6985 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1433 | Regression_eta: 0.0134 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9279 | Classification_binary: 2.6985 | Regression_cos_phi: 0.6097 | Regression_energy: 1.5698 | Regression_eta: 0.6567 | Regression_pt: 1.6343 | Regression_sin_phi: 0.5216 | LR: 7.57e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7458s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2051 | max_loss: 0.0032 | max_backward: 0.2864 | max_optimizer_step: 0.2175 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4210.6 | uss_MB: 3447.2 | child_rss_MB: 4004.1 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:21:50,569] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:31,806] INFO : Step 17500/50000 rank0 | Train Loss: 8.7239 | Train Loss Unweighted: 3.0728 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.7257 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1509 | Regression_eta: 0.0117 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9482 | Classification_binary: 2.7257 | Regression_cos_phi: 0.5798 | Regression_energy: 1.6530 | Regression_eta: 0.5761 | Regression_pt: 1.7819 | Regression_sin_phi: 0.4591 | LR: 7.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6158s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1504 | max_loss: 0.0033 | max_backward: 0.2867 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.2893 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4211.6 | uss_MB: 3447.2 | child_rss_MB: 4000.9 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 13:22:32,028] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:13,330] INFO : Step 17600/50000 rank0 | Train Loss: 7.9389 | Train Loss Unweighted: 2.9411 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6231 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1399 | Regression_eta: 0.0089 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8998 | Classification_binary: 2.6231 | Regression_cos_phi: 0.4321 | Regression_energy: 1.5325 | Regression_eta: 0.4359 | Regression_pt: 1.6557 | Regression_sin_phi: 0.3598 | LR: 7.52e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5149s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0440 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.2025 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.2859 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4211.6 | uss_MB: 3445.2 | child_rss_MB: 4001.6 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:23:13,560] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:55,206] INFO : Step 17700/50000 rank0 | Train Loss: 8.4270 | Train Loss Unweighted: 3.0586 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7308 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1445 | Regression_eta: 0.0131 | Regression_pt: 0.1370 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8838 | Classification_binary: 2.7308 | Regression_cos_phi: 0.5019 | Regression_energy: 1.5830 | Regression_eta: 0.6439 | Regression_pt: 1.6435 | Regression_sin_phi: 0.4401 | LR: 7.49e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6206s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.3074 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4211.6 | uss_MB: 3445.2 | child_rss_MB: 4003.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:23:55,421] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:36,700] INFO : Step 17800/50000 rank0 | Train Loss: 8.0407 | Train Loss Unweighted: 2.9458 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.6244 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1416 | Regression_eta: 0.0101 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9029 | Classification_binary: 2.6244 | Regression_cos_phi: 0.4337 | Regression_energy: 1.5519 | Regression_eta: 0.4981 | Regression_pt: 1.6566 | Regression_sin_phi: 0.3731 | LR: 7.47e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5503s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.2378 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4211.5 | uss_MB: 3444.2 | child_rss_MB: 4003.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:24:36,911] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:17,542] INFO : Step 17900/50000 rank0 | Train Loss: 7.9466 | Train Loss Unweighted: 2.8611 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5574 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1325 | Regression_eta: 0.0130 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8278 | Classification_binary: 2.5574 | Regression_cos_phi: 0.5040 | Regression_energy: 1.4522 | Regression_eta: 0.6379 | Regression_pt: 1.5105 | Regression_sin_phi: 0.4568 | LR: 7.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6386s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.3111 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3905 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4212.5 | uss_MB: 3445.0 | child_rss_MB: 4000.8 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:25:17,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:57,870] INFO : Step 18000/50000 rank0 | Train Loss: 9.0771 | Train Loss Unweighted: 3.0841 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7124 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1615 | Regression_eta: 0.0155 | Regression_pt: 0.1598 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9163 | Classification_binary: 2.7124 | Regression_cos_phi: 0.4795 | Regression_energy: 1.7697 | Regression_eta: 0.7630 | Regression_pt: 1.9166 | Regression_sin_phi: 0.5196 | LR: 7.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4841s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1510 | max_loss: 0.0032 | max_backward: 0.2020 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4212.5 | uss_MB: 3445.1 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:25:58,086] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:58,107] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 13:25:58,288] INFO : Running validation on rank0 +rank0 [2026-10-28 13:25:58,288] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:25:58,288] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:25:59,676] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 13:25:59,823] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:25:59,954] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:26:00,029] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:26:00,099] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:26:00,168] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:26:00,238] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:26:00,307] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:26:00,418] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:26:00,549] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:26:00,679] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:26:00,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:26:00,820] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:26:00,887] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:26:00,957] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:26:01,027] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:26:01,137] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:26:01,144] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:26:01,146] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:26:01,149] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:26:01,150] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:26:01,153] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:26:01,154] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:26:01,157] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:26:01,159] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:26:01,160] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:26:01,163] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:26:01,165] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:26:01,166] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:26:01,169] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:26:01,170] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:26:01,173] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:26:08,841] INFO : VALIDATION | Step=18000/50000 | Train Loss=9.0771 | Valid Loss=8.4217 | Train Loss Unweighted=3.0841 | Valid Loss Unweighted=2.9995 | Stale=0 | ETA=237.5m +rank0 [2026-10-28 13:26:08,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:08,926] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:26:08,930] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:09,093] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:11,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:11,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:12,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:13,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:13,311] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:26:13,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:13,394] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:26:13,397] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:13,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:15,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:15,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:16,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:17,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:17,442] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:26:17,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:17,526] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:26:17,530] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:26:17,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:19,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:20,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:20,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:21,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:21,553] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:26:21,561] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:21,817] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:22,220] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:24,130] INFO : Plotted jets +rank0 [2026-10-28 13:26:25,213] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:25,246] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:25,473] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:25,862] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:27,252] INFO : Plotted jets +rank0 [2026-10-28 13:26:28,358] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:28,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:28,603] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:28,966] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:30,343] INFO : Plotted jets +rank0 [2026-10-28 13:26:31,396] INFO : Plotted jet ratio +rank0 [2026-10-28 13:27:13,795] INFO : Step 18100/50000 rank0 | Train Loss: 8.0182 | Train Loss Unweighted: 2.9006 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5822 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1426 | Regression_eta: 0.0096 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9449 | Classification_binary: 2.5822 | Regression_cos_phi: 0.4793 | Regression_energy: 1.5621 | Regression_eta: 0.4700 | Regression_pt: 1.5944 | Regression_sin_phi: 0.3852 | LR: 7.39e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.6557s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.3084 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4427.3 | uss_MB: 3661.9 | child_rss_MB: 4007.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:27:14,005] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:27:55,277] INFO : Step 18200/50000 rank0 | Train Loss: 8.5521 | Train Loss Unweighted: 3.0129 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6804 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1445 | Regression_eta: 0.0128 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8832 | Classification_binary: 2.6804 | Regression_cos_phi: 0.5165 | Regression_energy: 1.5829 | Regression_eta: 0.6271 | Regression_pt: 1.6733 | Regression_sin_phi: 0.5887 | LR: 7.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.3097 | max_optimizer_step: 0.1246 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4283.5 | uss_MB: 3517.3 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:27:55,491] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:28:36,731] INFO : Step 18300/50000 rank0 | Train Loss: 8.4956 | Train Loss Unweighted: 2.9940 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6410 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1570 | Regression_eta: 0.0095 | Regression_pt: 0.1534 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8549 | Classification_binary: 2.6410 | Regression_cos_phi: 0.4901 | Regression_energy: 1.7205 | Regression_eta: 0.4672 | Regression_pt: 1.8392 | Regression_sin_phi: 0.4827 | LR: 7.34e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1060 | max_forward: 0.1039 | max_loss: 0.0032 | max_backward: 0.3929 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3904 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4284.5 | uss_MB: 3520.3 | child_rss_MB: 4007.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:28:36,946] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:17,798] INFO : Step 18400/50000 rank0 | Train Loss: 8.6009 | Train Loss Unweighted: 3.0080 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6734 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1478 | Regression_eta: 0.0154 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9629 | Classification_binary: 2.6734 | Regression_cos_phi: 0.4904 | Regression_energy: 1.6192 | Regression_eta: 0.7574 | Regression_pt: 1.6353 | Regression_sin_phi: 0.4624 | LR: 7.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5408s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1826 | max_loss: 0.0033 | max_backward: 0.2024 | max_optimizer_step: 0.1235 | max_tensorboard_logging: 0.2798 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4284.5 | uss_MB: 3518.4 | child_rss_MB: 4007.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:29:18,014] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:59,103] INFO : Step 18500/50000 rank0 | Train Loss: 8.0647 | Train Loss Unweighted: 2.9308 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6198 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1335 | Regression_eta: 0.0090 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9516 | Classification_binary: 2.6198 | Regression_cos_phi: 0.5532 | Regression_energy: 1.4624 | Regression_eta: 0.4406 | Regression_pt: 1.5993 | Regression_sin_phi: 0.4379 | LR: 7.29e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6184s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.2806 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3867 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4284.5 | uss_MB: 3518.5 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:29:59,314] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:30:40,886] INFO : Step 18600/50000 rank0 | Train Loss: 7.8086 | Train Loss Unweighted: 2.9032 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6069 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1276 | Regression_eta: 0.0086 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8893 | Classification_binary: 2.6069 | Regression_cos_phi: 0.4725 | Regression_energy: 1.3987 | Regression_eta: 0.4238 | Regression_pt: 1.5134 | Regression_sin_phi: 0.5040 | LR: 7.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5630s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1244 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4284.5 | uss_MB: 3518.6 | child_rss_MB: 4004.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 13:30:41,111] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:31:22,192] INFO : Step 18700/50000 rank0 | Train Loss: 8.4602 | Train Loss Unweighted: 2.9952 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6599 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1460 | Regression_eta: 0.0103 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8939 | Classification_binary: 2.6599 | Regression_cos_phi: 0.5413 | Regression_energy: 1.5992 | Regression_eta: 0.5050 | Regression_pt: 1.7214 | Regression_sin_phi: 0.5395 | LR: 7.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5398s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1854 | max_loss: 0.0033 | max_backward: 0.2026 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4284.5 | uss_MB: 3518.6 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:31:22,407] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:03,686] INFO : Step 18800/50000 rank0 | Train Loss: 8.4335 | Train Loss Unweighted: 2.8989 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5783 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1374 | Regression_eta: 0.0206 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8401 | Classification_binary: 2.5783 | Regression_cos_phi: 0.4429 | Regression_energy: 1.5054 | Regression_eta: 1.0118 | Regression_pt: 1.5642 | Regression_sin_phi: 0.4907 | LR: 7.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5503s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1848 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2981 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4282.6 | uss_MB: 3517.4 | child_rss_MB: 4007.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:32:03,910] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:44,551] INFO : Step 18900/50000 rank0 | Train Loss: 8.5882 | Train Loss Unweighted: 3.0248 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6732 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1575 | Regression_eta: 0.0113 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8828 | Classification_binary: 2.6732 | Regression_cos_phi: 0.5376 | Regression_energy: 1.7258 | Regression_eta: 0.5568 | Regression_pt: 1.7931 | Regression_sin_phi: 0.4189 | LR: 7.18e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6011s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2639 | max_optimizer_step: 0.1240 | max_tensorboard_logging: 0.3932 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4282.6 | uss_MB: 3518.2 | child_rss_MB: 3999.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:32:44,763] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:33:25,385] INFO : Step 19000/50000 rank0 | Train Loss: 8.2713 | Train Loss Unweighted: 3.0333 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.7099 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1421 | Regression_eta: 0.0107 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9244 | Classification_binary: 2.7099 | Regression_cos_phi: 0.5212 | Regression_energy: 1.5568 | Regression_eta: 0.5281 | Regression_pt: 1.6444 | Regression_sin_phi: 0.3865 | LR: 7.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6455s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0032 | max_backward: 0.3107 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.3946 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4282.6 | uss_MB: 3518.2 | child_rss_MB: 3999.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:33:25,596] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:33:25,708] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 13:33:25,886] INFO : Running validation on rank0 +rank0 [2026-10-28 13:33:25,886] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:33:25,886] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:33:27,214] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 13:33:27,360] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:33:27,491] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:33:27,565] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:33:27,634] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:33:27,703] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:33:27,773] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:33:27,842] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:33:27,954] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:33:28,085] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:33:28,216] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:33:28,287] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:33:28,356] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:33:28,424] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:33:28,494] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:33:28,562] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:33:28,672] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:33:28,679] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:33:28,682] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:33:28,684] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:33:28,686] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:33:28,689] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:33:28,692] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:33:28,693] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:33:28,696] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:33:28,697] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:33:28,700] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:33:28,702] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:33:28,703] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:33:28,706] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:33:28,708] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:33:28,709] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:33:36,311] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.2713 | Valid Loss=8.3138 | Train Loss Unweighted=3.0333 | Valid Loss Unweighted=2.9580 | Stale=0 | ETA=230.1m +rank0 [2026-10-28 13:33:36,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:33:36,384] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:33:36,387] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:33:36,546] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:33:38,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:33:39,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:33:39,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:33:40,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:33:40,555] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:33:40,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:33:40,699] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:33:40,702] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:33:40,862] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:33:42,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:33:43,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:33:44,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:33:44,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:33:44,784] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:33:44,787] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:33:44,879] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:33:44,892] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:33:45,058] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:33:46,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:33:47,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:33:48,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:33:48,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:33:48,930] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:33:48,938] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:33:49,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:33:49,598] INFO : Plotted number of elements +rank0 [2026-10-28 13:33:51,030] INFO : Plotted jets +rank0 [2026-10-28 13:33:52,635] INFO : Plotted jet ratio +rank0 [2026-10-28 13:33:52,669] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:33:52,901] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:33:53,289] INFO : Plotted number of elements +rank0 [2026-10-28 13:33:54,677] INFO : Plotted jets +rank0 [2026-10-28 13:33:55,789] INFO : Plotted jet ratio +rank0 [2026-10-28 13:33:55,825] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 13:33:56,036] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 13:33:56,405] INFO : Plotted number of elements +rank0 [2026-10-28 13:33:57,779] INFO : Plotted jets +rank0 [2026-10-28 13:33:58,847] INFO : Plotted jet ratio +rank0 [2026-10-28 13:34:40,805] INFO : Step 19100/50000 rank0 | Train Loss: 8.3717 | Train Loss Unweighted: 3.0004 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6775 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1424 | Regression_eta: 0.0111 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9342 | Classification_binary: 2.6775 | Regression_cos_phi: 0.6129 | Regression_energy: 1.5606 | Regression_eta: 0.5474 | Regression_pt: 1.6010 | Regression_sin_phi: 0.4381 | LR: 7.13e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5816s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1817 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1261 | max_tensorboard_logging: 0.2966 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4483.8 | uss_MB: 3718.7 | child_rss_MB: 3998.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:34:41,020] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:35:21,801] INFO : Step 19200/50000 rank0 | Train Loss: 8.0526 | Train Loss Unweighted: 3.0095 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6838 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1448 | Regression_eta: 0.0091 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8346 | Classification_binary: 2.6838 | Regression_cos_phi: 0.4501 | Regression_energy: 1.5869 | Regression_eta: 0.4483 | Regression_pt: 1.7012 | Regression_sin_phi: 0.3478 | LR: 7.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7549s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2102 | max_loss: 0.0033 | max_backward: 0.2863 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.3150 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 4359.3 | uss_MB: 3595.6 | child_rss_MB: 3994.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:35:22,013] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:02,624] INFO : Step 19300/50000 rank0 | Train Loss: 8.2186 | Train Loss Unweighted: 2.9027 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5797 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1411 | Regression_eta: 0.0117 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8450 | Classification_binary: 2.5797 | Regression_cos_phi: 0.5185 | Regression_energy: 1.5460 | Regression_eta: 0.5770 | Regression_pt: 1.6362 | Regression_sin_phi: 0.5164 | LR: 7.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6707s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1810 | max_loss: 0.0033 | max_backward: 0.2868 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.2974 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4359.3 | uss_MB: 3596.7 | child_rss_MB: 3997.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:36:02,842] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:44,169] INFO : Step 19400/50000 rank0 | Train Loss: 8.6481 | Train Loss Unweighted: 3.0517 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7070 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1510 | Regression_eta: 0.0107 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0218 | Classification_binary: 2.7070 | Regression_cos_phi: 0.5424 | Regression_energy: 1.6551 | Regression_eta: 0.5260 | Regression_pt: 1.7543 | Regression_sin_phi: 0.4414 | LR: 7.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6144s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1656 | max_loss: 0.0032 | max_backward: 0.2024 | max_optimizer_step: 0.2246 | max_tensorboard_logging: 0.2779 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4359.3 | uss_MB: 3595.0 | child_rss_MB: 3999.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:36:44,393] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:37:24,989] INFO : Step 19500/50000 rank0 | Train Loss: 7.8863 | Train Loss Unweighted: 2.8800 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5720 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1342 | Regression_eta: 0.0097 | Regression_pt: 0.1316 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8782 | Classification_binary: 2.5720 | Regression_cos_phi: 0.5032 | Regression_energy: 1.4703 | Regression_eta: 0.4785 | Regression_pt: 1.5785 | Regression_sin_phi: 0.4057 | LR: 7.02e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.5112s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0461 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.2029 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4359.3 | uss_MB: 3595.1 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:37:25,215] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:38:06,429] INFO : Step 19600/50000 rank0 | Train Loss: 8.2227 | Train Loss Unweighted: 2.8989 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5863 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1338 | Regression_eta: 0.0155 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9282 | Classification_binary: 2.5863 | Regression_cos_phi: 0.4953 | Regression_energy: 1.4661 | Regression_eta: 0.7635 | Regression_pt: 1.5515 | Regression_sin_phi: 0.4319 | LR: 7.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5576s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0903 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.2034 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4359.3 | uss_MB: 3595.1 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:38:06,661] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:38:47,526] INFO : Step 19700/50000 rank0 | Train Loss: 8.5903 | Train Loss Unweighted: 2.9633 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6228 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1497 | Regression_eta: 0.0159 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8554 | Classification_binary: 2.6228 | Regression_cos_phi: 0.4766 | Regression_energy: 1.6406 | Regression_eta: 0.7834 | Regression_pt: 1.6966 | Regression_sin_phi: 0.5151 | LR: 6.97e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6493s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0033 | max_backward: 0.3060 | max_optimizer_step: 0.1276 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4359.3 | uss_MB: 3595.2 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:38:47,742] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:39:29,326] INFO : Step 19800/50000 rank0 | Train Loss: 9.3358 | Train Loss Unweighted: 2.9734 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6099 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1494 | Regression_eta: 0.0261 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9358 | Classification_binary: 2.6099 | Regression_cos_phi: 0.5500 | Regression_energy: 1.6365 | Regression_eta: 1.2815 | Regression_pt: 1.8262 | Regression_sin_phi: 0.4960 | LR: 6.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1707 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4357.4 | uss_MB: 3593.4 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:39:29,552] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:10,110] INFO : Step 19900/50000 rank0 | Train Loss: 8.1791 | Train Loss Unweighted: 2.8765 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5607 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1368 | Regression_eta: 0.0148 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8553 | Classification_binary: 2.5607 | Regression_cos_phi: 0.4987 | Regression_energy: 1.4988 | Regression_eta: 0.7298 | Regression_pt: 1.5748 | Regression_sin_phi: 0.4610 | LR: 6.92e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0682 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.1612 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4357.4 | uss_MB: 3594.8 | child_rss_MB: 3995.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:40:10,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:50,601] INFO : Step 20000/50000 rank0 | Train Loss: 9.3311 | Train Loss Unweighted: 3.0955 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7224 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1645 | Regression_eta: 0.0203 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8982 | Classification_binary: 2.7224 | Regression_cos_phi: 0.5752 | Regression_energy: 1.8020 | Regression_eta: 0.9989 | Regression_pt: 1.8335 | Regression_sin_phi: 0.5008 | LR: 6.89e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5339s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0448 | max_backward: 0.2031 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.3444 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4357.4 | uss_MB: 3594.9 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:40:50,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:50,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 13:40:51,035] INFO : Running validation on rank0 +rank0 [2026-10-28 13:40:51,035] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:40:51,035] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:40:52,333] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 13:40:52,479] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:40:52,611] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:40:52,685] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:40:52,755] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:40:52,823] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:40:52,893] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:40:52,962] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:40:53,076] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:40:53,208] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:40:53,339] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:40:53,410] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:40:53,480] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:40:53,548] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:40:53,617] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:40:53,686] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:40:53,795] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:40:53,802] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:40:53,805] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:40:53,808] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:40:53,810] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:40:53,811] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:40:53,814] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:40:53,817] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:40:53,819] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:40:53,821] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:40:53,823] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:40:53,824] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:40:53,827] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:40:53,830] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:40:53,832] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:40:53,834] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:41:01,275] INFO : VALIDATION | Step=20000/50000 | Train Loss=9.3311 | Valid Loss=8.2836 | Train Loss Unweighted=3.0955 | Valid Loss Unweighted=2.9411 | Stale=0 | ETA=222.7m +rank0 [2026-10-28 13:41:01,285] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:01,344] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:41:01,348] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:41:01,509] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:41:03,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:41:04,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:41:04,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:41:05,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:41:05,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:41:05,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:05,675] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:41:05,678] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:41:05,773] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:41:07,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:41:08,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:41:09,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:41:09,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:41:09,731] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:41:09,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:09,791] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:41:09,794] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:41:09,889] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:41:11,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:41:12,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:41:12,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:41:13,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:41:13,780] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:41:13,788] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:41:14,046] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:41:14,453] INFO : Plotted number of elements +rank0 [2026-10-28 13:41:15,898] INFO : Plotted jets +rank0 [2026-10-28 13:41:17,504] INFO : Plotted jet ratio +rank0 [2026-10-28 13:41:17,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:41:17,764] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:41:18,152] INFO : Plotted number of elements +rank0 [2026-10-28 13:41:19,538] INFO : Plotted jets +rank0 [2026-10-28 13:41:20,647] INFO : Plotted jet ratio +rank0 [2026-10-28 13:41:20,680] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 13:41:20,889] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 13:41:21,251] INFO : Plotted number of elements +rank0 [2026-10-28 13:41:22,627] INFO : Plotted jets +rank0 [2026-10-28 13:41:23,689] INFO : Plotted jet ratio +rank0 [2026-10-28 13:42:04,446] INFO : Step 20100/50000 rank0 | Train Loss: 9.0107 | Train Loss Unweighted: 3.0371 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6994 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1460 | Regression_eta: 0.0205 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8264 | Classification_binary: 2.6994 | Regression_cos_phi: 0.8001 | Regression_energy: 1.5995 | Regression_eta: 1.0060 | Regression_pt: 1.6153 | Regression_sin_phi: 0.4640 | LR: 6.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6701s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3080 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3886 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4549.9 | uss_MB: 3789.3 | child_rss_MB: 3997.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:42:04,658] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:42:45,945] INFO : Step 20200/50000 rank0 | Train Loss: 8.2460 | Train Loss Unweighted: 2.9560 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.6185 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1498 | Regression_eta: 0.0097 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8676 | Classification_binary: 2.6185 | Regression_cos_phi: 0.4873 | Regression_energy: 1.6416 | Regression_eta: 0.4763 | Regression_pt: 1.7500 | Regression_sin_phi: 0.4046 | LR: 6.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6838s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3080 | max_optimizer_step: 0.1270 | max_tensorboard_logging: 0.4077 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4425.2 | uss_MB: 3664.2 | child_rss_MB: 3995.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:42:46,159] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:43:26,769] INFO : Step 20300/50000 rank0 | Train Loss: 8.5452 | Train Loss Unweighted: 3.0755 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.7369 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1480 | Regression_eta: 0.0107 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9492 | Classification_binary: 2.7369 | Regression_cos_phi: 0.5304 | Regression_energy: 1.6222 | Regression_eta: 0.5280 | Regression_pt: 1.7369 | Regression_sin_phi: 0.4416 | LR: 6.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5638s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.2195 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4425.2 | uss_MB: 3665.4 | child_rss_MB: 3993.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:43:26,981] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:07,157] INFO : Step 20400/50000 rank0 | Train Loss: 8.5537 | Train Loss Unweighted: 3.0628 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7267 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1469 | Regression_eta: 0.0111 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8571 | Classification_binary: 2.7267 | Regression_cos_phi: 0.5053 | Regression_energy: 1.6094 | Regression_eta: 0.5459 | Regression_pt: 1.7164 | Regression_sin_phi: 0.5929 | LR: 6.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1512 | max_loss: 0.0034 | max_backward: 0.2870 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.2882 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4425.2 | uss_MB: 3668.1 | child_rss_MB: 3998.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:44:07,379] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:47,644] INFO : Step 20500/50000 rank0 | Train Loss: 8.2588 | Train Loss Unweighted: 2.9325 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6106 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1403 | Regression_eta: 0.0139 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9249 | Classification_binary: 2.6106 | Regression_cos_phi: 0.4682 | Regression_energy: 1.5373 | Regression_eta: 0.6843 | Regression_pt: 1.6123 | Regression_sin_phi: 0.4212 | LR: 6.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.3090 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.3749 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4425.2 | uss_MB: 3666.0 | child_rss_MB: 3996.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:44:47,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:45:29,320] INFO : Step 20600/50000 rank0 | Train Loss: 7.5367 | Train Loss Unweighted: 2.8990 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6025 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1308 | Regression_eta: 0.0068 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8990 | Classification_binary: 2.6025 | Regression_cos_phi: 0.3784 | Regression_energy: 1.4331 | Regression_eta: 0.3341 | Regression_pt: 1.5433 | Regression_sin_phi: 0.3464 | LR: 6.73e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5529s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0377 | max_backward: 0.2025 | max_optimizer_step: 0.1246 | max_tensorboard_logging: 0.3552 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4425.2 | uss_MB: 3666.0 | child_rss_MB: 3999.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:45:29,534] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:46:10,648] INFO : Step 20700/50000 rank0 | Train Loss: 8.3612 | Train Loss Unweighted: 2.9612 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6405 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1388 | Regression_eta: 0.0132 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9442 | Classification_binary: 2.6405 | Regression_cos_phi: 0.5144 | Regression_energy: 1.5211 | Regression_eta: 0.6488 | Regression_pt: 1.5973 | Regression_sin_phi: 0.4948 | LR: 6.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6531s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3075 | max_optimizer_step: 0.1246 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4425.2 | uss_MB: 3665.9 | child_rss_MB: 3998.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:46:10,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:46:52,324] INFO : Step 20800/50000 rank0 | Train Loss: 8.1590 | Train Loss Unweighted: 3.0229 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.7080 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1389 | Regression_eta: 0.0118 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8471 | Classification_binary: 2.7080 | Regression_cos_phi: 0.4731 | Regression_energy: 1.5224 | Regression_eta: 0.5805 | Regression_pt: 1.5848 | Regression_sin_phi: 0.4431 | LR: 6.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7314s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0032 | max_backward: 0.3787 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4424.2 | uss_MB: 3666.4 | child_rss_MB: 3999.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:46:52,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:47:34,076] INFO : Step 20900/50000 rank0 | Train Loss: 8.3585 | Train Loss Unweighted: 3.0257 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7012 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1422 | Regression_eta: 0.0109 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9429 | Classification_binary: 2.7012 | Regression_cos_phi: 0.5138 | Regression_energy: 1.5582 | Regression_eta: 0.5367 | Regression_pt: 1.6349 | Regression_sin_phi: 0.4710 | LR: 6.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6294s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0031 | max_backward: 0.2970 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4424.2 | uss_MB: 3665.5 | child_rss_MB: 4000.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:47:34,291] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:14,554] INFO : Step 21000/50000 rank0 | Train Loss: 8.6959 | Train Loss Unweighted: 2.9541 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6030 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1540 | Regression_eta: 0.0129 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8359 | Classification_binary: 2.6030 | Regression_cos_phi: 0.5367 | Regression_energy: 1.6879 | Regression_eta: 0.6348 | Regression_pt: 1.7844 | Regression_sin_phi: 0.6132 | LR: 6.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5863s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1187 | max_loss: 0.0032 | max_backward: 0.2870 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4425.2 | uss_MB: 3668.0 | child_rss_MB: 4000.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:48:14,774] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:14,795] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 13:48:14,966] INFO : Running validation on rank0 +rank0 [2026-10-28 13:48:14,966] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:48:14,966] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:48:16,282] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 13:48:16,430] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:48:16,563] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:48:16,637] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:48:16,706] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:48:16,775] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:48:16,845] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:48:16,914] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:48:17,026] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:48:17,158] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:48:17,291] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:48:17,362] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:48:17,431] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:48:17,499] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:48:17,570] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:48:17,639] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:48:17,754] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:48:17,762] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:48:17,764] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:48:17,767] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:48:17,770] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:48:17,772] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:48:17,774] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:48:17,777] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:48:17,780] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:48:17,782] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:48:17,783] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:48:17,786] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:48:17,789] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:48:17,791] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:48:17,792] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:48:17,795] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:48:25,485] INFO : VALIDATION | Step=21000/50000 | Train Loss=8.6959 | Valid Loss=8.1990 | Train Loss Unweighted=2.9541 | Valid Loss Unweighted=2.9126 | Stale=0 | ETA=215.2m +rank0 [2026-10-28 13:48:25,495] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:48:25,564] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:48:25,567] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:48:25,691] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:48:27,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:48:28,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:48:29,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:48:29,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:48:29,887] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:48:29,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:48:29,946] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:48:29,948] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:48:30,047] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:48:31,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:48:32,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:48:33,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:48:33,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:48:34,118] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:48:34,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:48:34,175] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:48:34,178] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:48:34,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:48:36,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:48:36,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:48:37,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:48:38,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:48:38,229] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:48:38,236] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:48:38,490] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:48:38,893] INFO : Plotted number of elements +rank0 [2026-10-28 13:48:40,329] INFO : Plotted jets +rank0 [2026-10-28 13:48:41,411] INFO : Plotted jet ratio +rank0 [2026-10-28 13:48:41,446] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:48:41,672] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:48:42,560] INFO : Plotted number of elements +rank0 [2026-10-28 13:48:43,947] INFO : Plotted jets +rank0 [2026-10-28 13:48:45,056] INFO : Plotted jet ratio +rank0 [2026-10-28 13:48:45,090] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 13:48:45,296] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 13:48:45,655] INFO : Plotted number of elements +rank0 [2026-10-28 13:48:47,032] INFO : Plotted jets +rank0 [2026-10-28 13:48:48,098] INFO : Plotted jet ratio +rank0 [2026-10-28 13:49:29,617] INFO : Step 21100/50000 rank0 | Train Loss: 8.4728 | Train Loss Unweighted: 2.9789 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6550 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1400 | Regression_eta: 0.0128 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8869 | Classification_binary: 2.6550 | Regression_cos_phi: 0.6293 | Regression_energy: 1.5344 | Regression_eta: 0.6304 | Regression_pt: 1.6147 | Regression_sin_phi: 0.5222 | LR: 6.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5613s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0582 | max_backward: 0.2023 | max_optimizer_step: 0.1268 | max_tensorboard_logging: 0.3362 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4624.4 | uss_MB: 3865.5 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:49:29,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:49:34,829] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 13:49:34,829] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:50:10,546] INFO : Step 21200/50000 rank0 | Train Loss: 8.4097 | Train Loss Unweighted: 2.9738 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6443 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1448 | Regression_eta: 0.0136 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9466 | Classification_binary: 2.6443 | Regression_cos_phi: 0.4363 | Regression_energy: 1.5869 | Regression_eta: 0.6703 | Regression_pt: 1.6408 | Regression_sin_phi: 0.4846 | LR: 6.56e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5119s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0157 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3000 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4498.1 | uss_MB: 3738.2 | child_rss_MB: 4000.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:50:10,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:50:51,978] INFO : Step 21300/50000 rank0 | Train Loss: 7.7799 | Train Loss Unweighted: 2.8683 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5589 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1365 | Regression_eta: 0.0091 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.5589 | Regression_cos_phi: 0.4313 | Regression_energy: 1.4953 | Regression_eta: 0.4496 | Regression_pt: 1.5932 | Regression_sin_phi: 0.3915 | LR: 6.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6471s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0792 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3091 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.2740 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4498.1 | uss_MB: 3739.2 | child_rss_MB: 4021.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:50:52,194] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:51:33,694] INFO : Step 21400/50000 rank0 | Train Loss: 7.5827 | Train Loss Unweighted: 2.7862 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.4968 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1262 | Regression_eta: 0.0082 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9231 | Classification_binary: 2.4968 | Regression_cos_phi: 0.5027 | Regression_energy: 1.3827 | Regression_eta: 0.4033 | Regression_pt: 1.4553 | Regression_sin_phi: 0.4189 | LR: 6.51e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5741s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0802 | max_backward: 0.2030 | max_optimizer_step: 0.1279 | max_tensorboard_logging: 0.3275 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4498.1 | uss_MB: 3739.5 | child_rss_MB: 3999.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:51:33,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:52:14,262] INFO : Step 21500/50000 rank0 | Train Loss: 7.9749 | Train Loss Unweighted: 2.9532 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6541 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1280 | Regression_eta: 0.0142 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9367 | Classification_binary: 2.6541 | Regression_cos_phi: 0.4368 | Regression_energy: 1.4022 | Regression_eta: 0.6976 | Regression_pt: 1.4993 | Regression_sin_phi: 0.3480 | LR: 6.48e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5894s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.2340 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3798 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4498.1 | uss_MB: 3739.5 | child_rss_MB: 3996.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:52:14,473] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:52:54,910] INFO : Step 21600/50000 rank0 | Train Loss: 7.9957 | Train Loss Unweighted: 2.8753 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.5437 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1471 | Regression_eta: 0.0085 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8874 | Classification_binary: 2.5437 | Regression_cos_phi: 0.4589 | Regression_energy: 1.6114 | Regression_eta: 0.4174 | Regression_pt: 1.7395 | Regression_sin_phi: 0.3374 | LR: 6.45e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5815s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.2304 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4498.1 | uss_MB: 3739.6 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:52:55,122] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:53:35,490] INFO : Step 21700/50000 rank0 | Train Loss: 7.9409 | Train Loss Unweighted: 3.0024 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.6883 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1391 | Regression_eta: 0.0100 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7975 | Classification_binary: 2.6883 | Regression_cos_phi: 0.4199 | Regression_energy: 1.5244 | Regression_eta: 0.4931 | Regression_pt: 1.6272 | Regression_sin_phi: 0.3904 | LR: 6.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6657s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.3087 | max_optimizer_step: 0.1264 | max_tensorboard_logging: 0.3815 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4498.1 | uss_MB: 3739.6 | child_rss_MB: 3995.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:53:35,704] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:54:17,375] INFO : Step 21800/50000 rank0 | Train Loss: 8.1084 | Train Loss Unweighted: 2.9406 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6312 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1332 | Regression_eta: 0.0121 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9257 | Classification_binary: 2.6312 | Regression_cos_phi: 0.5164 | Regression_energy: 1.4596 | Regression_eta: 0.5966 | Regression_pt: 1.5596 | Regression_sin_phi: 0.4191 | LR: 6.40e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5395s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1844 | max_loss: 0.0035 | max_backward: 0.2023 | max_optimizer_step: 0.1231 | max_tensorboard_logging: 0.2928 | max_tensorboard_flush: 0.0013 | Memory: rss_MB: 4496.0 | uss_MB: 3738.2 | child_rss_MB: 3996.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:54:17,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:54:58,321] INFO : Step 21900/50000 rank0 | Train Loss: 8.5814 | Train Loss Unweighted: 2.9783 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6402 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1465 | Regression_eta: 0.0161 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8941 | Classification_binary: 2.6402 | Regression_cos_phi: 0.5331 | Regression_energy: 1.6054 | Regression_eta: 0.7939 | Regression_pt: 1.7039 | Regression_sin_phi: 0.4108 | LR: 6.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6091s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1581 | max_loss: 0.0032 | max_backward: 0.2023 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.2750 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4497.0 | uss_MB: 3739.1 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:54:58,554] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:39,990] INFO : Step 22000/50000 rank0 | Train Loss: 8.4155 | Train Loss Unweighted: 2.9393 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6007 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1488 | Regression_eta: 0.0114 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8913 | Classification_binary: 2.6007 | Regression_cos_phi: 0.5822 | Regression_energy: 1.6304 | Regression_eta: 0.5629 | Regression_pt: 1.7302 | Regression_sin_phi: 0.4178 | LR: 6.34e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6643s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3106 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3832 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4497.0 | uss_MB: 3739.3 | child_rss_MB: 3998.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:55:40,201] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:40,221] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 13:55:40,512] INFO : Running validation on rank0 +rank0 [2026-10-28 13:55:40,512] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:55:40,512] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:55:41,990] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 13:55:42,136] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:55:42,268] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:55:42,341] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:55:42,410] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:55:42,478] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:55:42,547] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:55:42,616] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:55:42,727] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:55:42,859] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:55:42,990] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:55:43,061] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:55:43,130] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:55:43,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:55:43,268] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:55:43,338] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:55:43,453] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:55:43,460] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:55:43,463] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:55:43,466] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:55:43,468] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:55:43,469] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:55:43,472] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:55:43,475] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:55:43,476] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:55:43,479] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:55:43,481] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:55:43,482] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:55:43,485] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:55:43,486] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:55:43,489] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:55:43,492] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:55:50,922] INFO : VALIDATION | Step=22000/50000 | Train Loss=8.4155 | Valid Loss=8.1094 | Train Loss Unweighted=2.9393 | Valid Loss Unweighted=2.8928 | Stale=0 | ETA=207.8m +rank0 [2026-10-28 13:55:50,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:51,031] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:55:51,034] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:55:51,225] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:55:53,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:55:53,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:55:54,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:55:55,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:55:55,352] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:55:55,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:55,429] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:55:55,432] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:55:55,511] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:55:57,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:55:57,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:55:58,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:55:59,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:55:59,301] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:55:59,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:59,369] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:55:59,372] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:55:59,474] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:56:01,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:56:01,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:56:02,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:56:03,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:56:03,428] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:56:03,452] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:56:03,714] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:56:04,119] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:05,555] INFO : Plotted jets +rank0 [2026-10-28 13:56:06,644] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:06,677] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:56:06,904] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:56:07,830] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:09,215] INFO : Plotted jets +rank0 [2026-10-28 13:56:10,319] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:10,351] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 13:56:10,558] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 13:56:10,917] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:12,286] INFO : Plotted jets +rank0 [2026-10-28 13:56:13,339] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:54,797] INFO : Step 22100/50000 rank0 | Train Loss: 7.7992 | Train Loss Unweighted: 2.8871 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5816 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1337 | Regression_eta: 0.0093 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8173 | Classification_binary: 2.5816 | Regression_cos_phi: 0.4777 | Regression_energy: 1.4646 | Regression_eta: 0.4557 | Regression_pt: 1.5755 | Regression_sin_phi: 0.4267 | LR: 6.32e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7047s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3113 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3874 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4693.8 | uss_MB: 3937.2 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:56:55,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:57:36,177] INFO : Step 22200/50000 rank0 | Train Loss: 8.1853 | Train Loss Unweighted: 2.9477 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6306 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1378 | Regression_eta: 0.0109 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8884 | Classification_binary: 2.6306 | Regression_cos_phi: 0.5101 | Regression_energy: 1.5098 | Regression_eta: 0.5346 | Regression_pt: 1.6078 | Regression_sin_phi: 0.5040 | LR: 6.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6613s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0014 | max_forward: 0.0780 | max_loss: 0.0031 | max_backward: 0.2807 | max_optimizer_step: 0.1276 | max_tensorboard_logging: 0.3808 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4693.8 | uss_MB: 3937.3 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:57:36,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:16,992] INFO : Step 22300/50000 rank0 | Train Loss: 8.4874 | Train Loss Unweighted: 2.9227 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.5993 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1427 | Regression_eta: 0.0146 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9553 | Classification_binary: 2.5993 | Regression_cos_phi: 0.4938 | Regression_energy: 1.5635 | Regression_eta: 0.7161 | Regression_pt: 1.5443 | Regression_sin_phi: 0.6151 | LR: 6.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5798s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1842 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4572.2 | uss_MB: 3816.2 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:58:17,209] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:58,234] INFO : Step 22400/50000 rank0 | Train Loss: 8.2947 | Train Loss Unweighted: 2.8942 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.5767 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1371 | Regression_eta: 0.0131 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0467 | Classification_binary: 2.5767 | Regression_cos_phi: 0.4954 | Regression_energy: 1.5026 | Regression_eta: 0.6429 | Regression_pt: 1.5630 | Regression_sin_phi: 0.4674 | LR: 6.23e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6526s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1046 | max_loss: 0.0934 | max_backward: 0.2868 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.2833 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4572.2 | uss_MB: 3818.4 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 13:58:58,447] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:59:39,074] INFO : Step 22500/50000 rank0 | Train Loss: 8.2244 | Train Loss Unweighted: 2.9399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6272 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1335 | Regression_eta: 0.0149 | Regression_pt: 0.1316 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8365 | Classification_binary: 2.6272 | Regression_cos_phi: 0.5829 | Regression_energy: 1.4624 | Regression_eta: 0.7343 | Regression_pt: 1.5787 | Regression_sin_phi: 0.4025 | LR: 6.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6642s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0034 | max_backward: 0.2025 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4572.2 | uss_MB: 3816.3 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:59:39,300] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:00:20,798] INFO : Step 22600/50000 rank0 | Train Loss: 8.0256 | Train Loss Unweighted: 2.8441 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5339 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1356 | Regression_eta: 0.0137 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8493 | Classification_binary: 2.5339 | Regression_cos_phi: 0.4901 | Regression_energy: 1.4855 | Regression_eta: 0.6715 | Regression_pt: 1.5415 | Regression_sin_phi: 0.4539 | LR: 6.18e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6862s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3096 | max_optimizer_step: 0.1296 | max_tensorboard_logging: 0.3901 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4572.2 | uss_MB: 3816.4 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:00:21,011] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:01,782] INFO : Step 22700/50000 rank0 | Train Loss: 8.4938 | Train Loss Unweighted: 2.9796 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6428 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1470 | Regression_eta: 0.0109 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9236 | Classification_binary: 2.6428 | Regression_cos_phi: 0.5413 | Regression_energy: 1.6103 | Regression_eta: 0.5353 | Regression_pt: 1.7146 | Regression_sin_phi: 0.5260 | LR: 6.15e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6714s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1361 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.3070 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 4572.2 | uss_MB: 3816.5 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:01:01,995] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:44,258] INFO : Step 22800/50000 rank0 | Train Loss: 7.9453 | Train Loss Unweighted: 2.9457 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6396 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1338 | Regression_eta: 0.0115 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8059 | Classification_binary: 2.6396 | Regression_cos_phi: 0.4581 | Regression_energy: 1.4661 | Regression_eta: 0.5635 | Regression_pt: 1.5564 | Regression_sin_phi: 0.4556 | LR: 6.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5691s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2442 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4572.2 | uss_MB: 3816.6 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:01:44,470] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:02:25,656] INFO : Step 22900/50000 rank0 | Train Loss: 8.8741 | Train Loss Unweighted: 2.9670 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6191 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1516 | Regression_eta: 0.0143 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9386 | Classification_binary: 2.6191 | Regression_cos_phi: 0.7624 | Regression_energy: 1.6608 | Regression_eta: 0.7041 | Regression_pt: 1.7213 | Regression_sin_phi: 0.4678 | LR: 6.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5532s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0404 | max_backward: 0.2024 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3394 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4572.1 | uss_MB: 3816.4 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:02:25,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:05,587] INFO : Step 23000/50000 rank0 | Train Loss: 7.5337 | Train Loss Unweighted: 2.7947 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5004 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1294 | Regression_eta: 0.0093 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8350 | Classification_binary: 2.5004 | Regression_cos_phi: 0.3952 | Regression_energy: 1.4174 | Regression_eta: 0.4548 | Regression_pt: 1.5025 | Regression_sin_phi: 0.4285 | LR: 6.06e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7219s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3919 | max_optimizer_step: 0.1277 | max_tensorboard_logging: 0.3436 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4572.1 | uss_MB: 3818.8 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:03:05,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:05,860] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 14:03:06,036] INFO : Running validation on rank0 +rank0 [2026-10-28 14:03:06,036] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:03:06,036] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:03:07,362] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 14:03:07,508] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:03:07,639] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:03:07,713] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:03:07,783] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:03:07,851] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:03:07,921] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:03:07,990] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:03:08,102] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:03:08,233] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:03:08,364] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:03:08,436] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:03:08,505] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:03:08,573] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:03:08,642] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:03:08,712] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:03:08,830] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:03:08,837] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:03:08,839] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:03:08,842] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:03:08,844] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:03:08,846] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:03:08,849] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:03:08,850] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:03:08,852] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:03:08,854] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:03:08,857] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:03:08,859] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:03:08,862] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:03:08,863] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:03:08,865] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:03:08,868] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:03:16,467] INFO : VALIDATION | Step=23000/50000 | Train Loss=7.5337 | Valid Loss=8.0745 | Train Loss Unweighted=2.7947 | Valid Loss Unweighted=2.8788 | Stale=0 | ETA=200.4m +rank0 [2026-10-28 14:03:16,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:03:16,546] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:03:16,550] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:03:16,713] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:03:18,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:03:19,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:03:20,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:03:20,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:03:20,749] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:03:20,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:03:20,808] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:03:20,811] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:03:20,913] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:03:22,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:03:23,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:03:24,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:03:24,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:03:24,787] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:03:24,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:03:24,844] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:03:24,847] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:03:24,950] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:03:26,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:03:27,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:03:28,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:03:28,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:03:28,937] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:03:28,944] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:03:29,209] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:03:29,611] INFO : Plotted number of elements +rank0 [2026-10-28 14:03:31,053] INFO : Plotted jets +rank0 [2026-10-28 14:03:32,657] INFO : Plotted jet ratio +rank0 [2026-10-28 14:03:32,689] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:03:32,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:03:33,312] INFO : Plotted number of elements +rank0 [2026-10-28 14:03:34,704] INFO : Plotted jets +rank0 [2026-10-28 14:03:35,819] INFO : Plotted jet ratio +rank0 [2026-10-28 14:03:35,850] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 14:03:36,057] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 14:03:36,419] INFO : Plotted number of elements +rank0 [2026-10-28 14:03:37,796] INFO : Plotted jets +rank0 [2026-10-28 14:03:38,856] INFO : Plotted jet ratio +rank0 [2026-10-28 14:04:20,498] INFO : Step 23100/50000 rank0 | Train Loss: 8.8554 | Train Loss Unweighted: 3.0243 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6825 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1455 | Regression_eta: 0.0186 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8274 | Classification_binary: 2.6825 | Regression_cos_phi: 0.6238 | Regression_energy: 1.5943 | Regression_eta: 0.9159 | Regression_pt: 1.7167 | Regression_sin_phi: 0.4948 | LR: 6.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6204s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2285 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.3880 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4756.7 | uss_MB: 4002.5 | child_rss_MB: 3998.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:04:20,712] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:00,737] INFO : Step 23200/50000 rank0 | Train Loss: 8.1110 | Train Loss Unweighted: 2.9762 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6645 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1355 | Regression_eta: 0.0101 | Regression_pt: 0.1316 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9519 | Classification_binary: 2.6645 | Regression_cos_phi: 0.5202 | Regression_energy: 1.4845 | Regression_eta: 0.4951 | Regression_pt: 1.5786 | Regression_sin_phi: 0.4161 | LR: 6.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1860 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.1321 | max_tensorboard_logging: 0.2906 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4756.7 | uss_MB: 4002.5 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:05:00,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:41,710] INFO : Step 23300/50000 rank0 | Train Loss: 7.5451 | Train Loss Unweighted: 2.8679 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5676 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1338 | Regression_eta: 0.0063 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8359 | Classification_binary: 2.5676 | Regression_cos_phi: 0.3915 | Regression_energy: 1.4658 | Regression_eta: 0.3093 | Regression_pt: 1.5595 | Regression_sin_phi: 0.4155 | LR: 5.98e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.6635s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3101 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3718 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4641.3 | uss_MB: 3887.3 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:05:41,924] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:06:22,175] INFO : Step 23400/50000 rank0 | Train Loss: 7.7281 | Train Loss Unweighted: 2.8641 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5596 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1325 | Regression_eta: 0.0089 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8765 | Classification_binary: 2.5596 | Regression_cos_phi: 0.4498 | Regression_energy: 1.4513 | Regression_eta: 0.4379 | Regression_pt: 1.5832 | Regression_sin_phi: 0.3698 | LR: 5.95e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7471s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1492 | max_loss: 0.0035 | max_backward: 0.3947 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4642.3 | uss_MB: 3889.4 | child_rss_MB: 4019.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:06:22,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:07:03,358] INFO : Step 23500/50000 rank0 | Train Loss: 7.8263 | Train Loss Unweighted: 2.9497 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6423 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1349 | Regression_eta: 0.0084 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9228 | Classification_binary: 2.6423 | Regression_cos_phi: 0.3874 | Regression_energy: 1.4783 | Regression_eta: 0.4111 | Regression_pt: 1.5879 | Regression_sin_phi: 0.3966 | LR: 5.92e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1787 | max_loss: 0.0035 | max_backward: 0.2028 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4642.3 | uss_MB: 3887.4 | child_rss_MB: 3996.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:07:03,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:07:46,001] INFO : Step 23600/50000 rank0 | Train Loss: 7.7871 | Train Loss Unweighted: 2.7430 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4451 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1294 | Regression_eta: 0.0142 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8649 | Classification_binary: 2.4451 | Regression_cos_phi: 0.4941 | Regression_energy: 1.4177 | Regression_eta: 0.6976 | Regression_pt: 1.4667 | Regression_sin_phi: 0.4010 | LR: 5.90e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6579s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1828 | max_loss: 0.0037 | max_backward: 0.2026 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.2771 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4642.3 | uss_MB: 3887.4 | child_rss_MB: 4000.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:07:46,227] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:08:27,765] INFO : Step 23700/50000 rank0 | Train Loss: 7.9213 | Train Loss Unweighted: 2.7989 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4817 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1376 | Regression_eta: 0.0121 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8508 | Classification_binary: 2.4817 | Regression_cos_phi: 0.4284 | Regression_energy: 1.5080 | Regression_eta: 0.5965 | Regression_pt: 1.6346 | Regression_sin_phi: 0.4212 | LR: 5.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7205s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1497 | max_loss: 0.0037 | max_backward: 0.3936 | max_optimizer_step: 0.1308 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4642.3 | uss_MB: 3889.5 | child_rss_MB: 4000.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 14:08:27,979] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:08,925] INFO : Step 23800/50000 rank0 | Train Loss: 8.6021 | Train Loss Unweighted: 2.8929 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5507 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1486 | Regression_eta: 0.0183 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8738 | Classification_binary: 2.5507 | Regression_cos_phi: 0.4889 | Regression_energy: 1.6285 | Regression_eta: 0.8974 | Regression_pt: 1.7073 | Regression_sin_phi: 0.4556 | LR: 5.84e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.5796s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1808 | max_loss: 0.0036 | max_backward: 0.2029 | max_optimizer_step: 0.1280 | max_tensorboard_logging: 0.2863 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4642.3 | uss_MB: 3887.3 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:09:09,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:50,059] INFO : Step 23900/50000 rank0 | Train Loss: 7.9644 | Train Loss Unweighted: 2.9233 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.6122 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1346 | Regression_eta: 0.0104 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8771 | Classification_binary: 2.6122 | Regression_cos_phi: 0.4395 | Regression_energy: 1.4752 | Regression_eta: 0.5120 | Regression_pt: 1.6059 | Regression_sin_phi: 0.4425 | LR: 5.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6693s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0032 | max_backward: 0.3107 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4641.8 | uss_MB: 3887.2 | child_rss_MB: 3999.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:09:50,270] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:10:31,697] INFO : Step 24000/50000 rank0 | Train Loss: 8.4389 | Train Loss Unweighted: 2.8459 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5152 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1424 | Regression_eta: 0.0132 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9725 | Classification_binary: 2.5152 | Regression_cos_phi: 0.5393 | Regression_energy: 1.5602 | Regression_eta: 0.6468 | Regression_pt: 1.6517 | Regression_sin_phi: 0.5530 | LR: 5.78e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5028s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0294 | max_forward: 0.0782 | max_loss: 0.0030 | max_backward: 0.2026 | max_optimizer_step: 0.1506 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4641.8 | uss_MB: 3887.3 | child_rss_MB: 3995.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:10:31,925] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:10:31,944] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 14:10:32,120] INFO : Running validation on rank0 +rank0 [2026-10-28 14:10:32,121] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:10:32,121] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:10:33,479] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 14:10:33,624] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:10:33,757] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:10:33,832] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:10:33,902] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:10:33,971] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:10:34,039] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:10:34,109] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:10:34,221] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:10:34,353] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:10:34,485] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:10:34,556] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:10:34,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:10:34,694] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:10:34,763] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:10:34,833] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:10:34,947] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:10:34,955] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:10:34,958] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:10:34,961] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:10:34,962] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:10:34,965] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:10:34,967] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:10:34,968] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:10:34,971] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:10:34,974] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:10:34,977] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:10:34,980] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:10:34,981] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:10:34,983] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:10:34,984] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:10:34,988] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:10:42,680] INFO : VALIDATION | Step=24000/50000 | Train Loss=8.4389 | Valid Loss=8.0030 | Train Loss Unweighted=2.8459 | Valid Loss Unweighted=2.8488 | Stale=0 | ETA=193.0m +rank0 [2026-10-28 14:10:42,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:10:42,790] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:10:42,794] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:10:42,933] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:10:45,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:10:45,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:10:46,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:10:47,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:10:47,235] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:10:47,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:10:47,322] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:10:47,325] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:10:47,399] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:10:49,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:10:49,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:10:50,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:10:51,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:10:51,347] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:10:51,350] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:10:51,417] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:10:51,420] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:10:51,507] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:10:53,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:10:54,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:10:54,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:10:55,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:10:55,510] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:10:55,517] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:10:55,779] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:10:56,183] INFO : Plotted number of elements +rank0 [2026-10-28 14:10:57,623] INFO : Plotted jets +rank0 [2026-10-28 14:10:59,221] INFO : Plotted jet ratio +rank0 [2026-10-28 14:10:59,255] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:10:59,484] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:10:59,872] INFO : Plotted number of elements +rank0 [2026-10-28 14:11:01,268] INFO : Plotted jets +rank0 [2026-10-28 14:11:02,382] INFO : Plotted jet ratio +rank0 [2026-10-28 14:11:02,436] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 14:11:02,641] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 14:11:03,004] INFO : Plotted number of elements +rank0 [2026-10-28 14:11:04,391] INFO : Plotted jets +rank0 [2026-10-28 14:11:05,451] INFO : Plotted jet ratio +rank0 [2026-10-28 14:11:46,396] INFO : Step 24100/50000 rank0 | Train Loss: 8.6500 | Train Loss Unweighted: 3.0110 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.6533 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1577 | Regression_eta: 0.0087 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8764 | Classification_binary: 2.6533 | Regression_cos_phi: 0.6090 | Regression_energy: 1.7277 | Regression_eta: 0.4299 | Regression_pt: 1.8715 | Regression_sin_phi: 0.4821 | LR: 5.75e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6891s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3107 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4853.3 | uss_MB: 4097.4 | child_rss_MB: 3995.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:11:46,610] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:12:27,174] INFO : Step 24200/50000 rank0 | Train Loss: 8.6277 | Train Loss Unweighted: 2.9879 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6362 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1549 | Regression_eta: 0.0119 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8555 | Classification_binary: 2.6362 | Regression_cos_phi: 0.6247 | Regression_energy: 1.6972 | Regression_eta: 0.5850 | Regression_pt: 1.8091 | Regression_sin_phi: 0.4199 | LR: 5.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6567s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.1074 | max_backward: 0.2024 | max_optimizer_step: 0.1319 | max_tensorboard_logging: 0.3219 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4853.3 | uss_MB: 4097.4 | child_rss_MB: 3996.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:12:27,385] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:08,745] INFO : Step 24300/50000 rank0 | Train Loss: 8.3283 | Train Loss Unweighted: 2.9707 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.6434 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1436 | Regression_eta: 0.0123 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8132 | Classification_binary: 2.6434 | Regression_cos_phi: 0.4931 | Regression_energy: 1.5735 | Regression_eta: 0.6036 | Regression_pt: 1.6588 | Regression_sin_phi: 0.5427 | LR: 5.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5311s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0706 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2026 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.2728 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4717.9 | uss_MB: 3961.6 | child_rss_MB: 3997.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:13:08,979] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:50,097] INFO : Step 24400/50000 rank0 | Train Loss: 8.3115 | Train Loss Unweighted: 2.8676 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5466 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1377 | Regression_eta: 0.0178 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9033 | Classification_binary: 2.5466 | Regression_cos_phi: 0.4707 | Regression_energy: 1.5092 | Regression_eta: 0.8735 | Regression_pt: 1.5931 | Regression_sin_phi: 0.4151 | LR: 5.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6907s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1846 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4717.9 | uss_MB: 3961.7 | child_rss_MB: 3999.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:13:50,325] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:14:32,439] INFO : Step 24500/50000 rank0 | Train Loss: 8.4930 | Train Loss Unweighted: 2.8531 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5234 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1426 | Regression_eta: 0.0137 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9471 | Classification_binary: 2.5234 | Regression_cos_phi: 0.6283 | Regression_energy: 1.5625 | Regression_eta: 0.6729 | Regression_pt: 1.6277 | Regression_sin_phi: 0.5312 | LR: 5.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6986s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3095 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.3829 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4717.9 | uss_MB: 3961.8 | child_rss_MB: 4001.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:14:32,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:15:13,182] INFO : Step 24600/50000 rank0 | Train Loss: 8.1257 | Train Loss Unweighted: 2.9536 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6371 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1365 | Regression_eta: 0.0109 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8797 | Classification_binary: 2.6371 | Regression_cos_phi: 0.4885 | Regression_energy: 1.4955 | Regression_eta: 0.5368 | Regression_pt: 1.6301 | Regression_sin_phi: 0.4579 | LR: 5.61e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7521s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1040 | max_loss: 0.0033 | max_backward: 0.3919 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.3547 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4717.9 | uss_MB: 3963.9 | child_rss_MB: 3999.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 14:15:13,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:15:55,184] INFO : Step 24700/50000 rank0 | Train Loss: 8.7131 | Train Loss Unweighted: 2.8448 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4946 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1524 | Regression_eta: 0.0133 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9652 | Classification_binary: 2.4946 | Regression_cos_phi: 0.6179 | Regression_energy: 1.6703 | Regression_eta: 0.6538 | Regression_pt: 1.7513 | Regression_sin_phi: 0.5599 | LR: 5.58e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8573s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0487 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.3965 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4717.9 | uss_MB: 3963.9 | child_rss_MB: 3997.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:15:55,399] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:16:36,607] INFO : Step 24800/50000 rank0 | Train Loss: 8.1072 | Train Loss Unweighted: 2.8987 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5808 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1395 | Regression_eta: 0.0084 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9021 | Classification_binary: 2.5808 | Regression_cos_phi: 0.5141 | Regression_energy: 1.5280 | Regression_eta: 0.4117 | Regression_pt: 1.6126 | Regression_sin_phi: 0.5581 | LR: 5.56e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5466s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0350 | max_backward: 0.2032 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3309 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4717.9 | uss_MB: 3961.9 | child_rss_MB: 4001.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:16:36,819] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:17,899] INFO : Step 24900/50000 rank0 | Train Loss: 8.1657 | Train Loss Unweighted: 2.9159 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.5940 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1405 | Regression_eta: 0.0134 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7835 | Classification_binary: 2.5940 | Regression_cos_phi: 0.4851 | Regression_energy: 1.5398 | Regression_eta: 0.6604 | Regression_pt: 1.6414 | Regression_sin_phi: 0.4613 | LR: 5.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6641s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.2927 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3846 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4718.5 | uss_MB: 3961.6 | child_rss_MB: 4001.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:17:18,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:58,814] INFO : Step 25000/50000 rank0 | Train Loss: 8.3112 | Train Loss Unweighted: 2.9163 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.5844 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1470 | Regression_eta: 0.0133 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7939 | Classification_binary: 2.5844 | Regression_cos_phi: 0.5939 | Regression_energy: 1.6102 | Regression_eta: 0.6555 | Regression_pt: 1.6768 | Regression_sin_phi: 0.3965 | LR: 5.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5832s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0918 | max_loss: 0.0034 | max_backward: 0.2028 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4718.5 | uss_MB: 3961.7 | child_rss_MB: 4004.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:17:59,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:59,060] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 14:17:59,235] INFO : Running validation on rank0 +rank0 [2026-10-28 14:17:59,235] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:17:59,235] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:18:00,588] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 14:18:00,732] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:18:00,865] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:18:00,940] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:18:01,009] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:18:01,077] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:18:01,146] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:18:01,216] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:18:01,328] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:18:01,460] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:18:01,592] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:18:01,665] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:18:01,734] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:18:01,802] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:18:01,871] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:18:01,941] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:18:02,053] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:18:02,060] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:18:02,063] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:18:02,064] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:18:02,067] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:18:02,070] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:18:02,072] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:18:02,074] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:18:02,078] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:18:02,081] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:18:02,083] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:18:02,086] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:18:02,090] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:18:02,091] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:18:02,094] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:18:02,096] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:18:09,649] INFO : VALIDATION | Step=25000/50000 | Train Loss=8.3112 | Valid Loss=7.9802 | Train Loss Unweighted=2.9163 | Valid Loss Unweighted=2.8465 | Stale=0 | ETA=185.6m +rank0 [2026-10-28 14:18:09,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:18:09,716] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:18:09,719] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:18:09,881] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:18:11,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:18:12,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:18:13,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:18:13,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:18:14,125] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:18:14,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:18:14,186] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:18:14,189] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:18:14,289] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:18:16,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:18:16,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:18:17,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:18:18,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:18:18,370] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:18:18,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:18:18,439] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:18:18,442] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:18:18,533] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:18:20,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:18:21,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:18:21,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:18:22,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:18:22,495] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:18:22,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:18:22,761] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:18:23,166] INFO : Plotted number of elements +rank0 [2026-10-28 14:18:24,603] INFO : Plotted jets +rank0 [2026-10-28 14:18:26,208] INFO : Plotted jet ratio +rank0 [2026-10-28 14:18:26,239] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:18:26,469] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:18:26,856] INFO : Plotted number of elements +rank0 [2026-10-28 14:18:28,242] INFO : Plotted jets +rank0 [2026-10-28 14:18:29,356] INFO : Plotted jet ratio +rank0 [2026-10-28 14:18:29,388] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 14:18:29,592] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 14:18:29,955] INFO : Plotted number of elements +rank0 [2026-10-28 14:18:31,331] INFO : Plotted jets +rank0 [2026-10-28 14:18:32,391] INFO : Plotted jet ratio +rank0 [2026-10-28 14:19:13,749] INFO : Step 25100/50000 rank0 | Train Loss: 7.7500 | Train Loss Unweighted: 2.7268 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4317 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1255 | Regression_eta: 0.0127 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9443 | Classification_binary: 2.4317 | Regression_cos_phi: 0.4815 | Regression_energy: 1.3746 | Regression_eta: 0.6224 | Regression_pt: 1.4766 | Regression_sin_phi: 0.4188 | LR: 5.47e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0124 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.3037 | max_optimizer_step: 0.1324 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4926.6 | uss_MB: 4169.8 | child_rss_MB: 4019.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:19:13,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:19:54,916] INFO : Step 25200/50000 rank0 | Train Loss: 7.7665 | Train Loss Unweighted: 2.8899 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5899 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1320 | Regression_eta: 0.0095 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7971 | Classification_binary: 2.5899 | Regression_cos_phi: 0.5207 | Regression_energy: 1.4465 | Regression_eta: 0.4683 | Regression_pt: 1.5263 | Regression_sin_phi: 0.4178 | LR: 5.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1047 | max_loss: 0.0373 | max_backward: 0.2868 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4926.6 | uss_MB: 4171.9 | child_rss_MB: 4002.7 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 14:19:55,135] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:20:36,937] INFO : Step 25300/50000 rank0 | Train Loss: 8.1388 | Train Loss Unweighted: 2.7887 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4558 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1469 | Regression_eta: 0.0114 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8217 | Classification_binary: 2.4558 | Regression_cos_phi: 0.5204 | Regression_energy: 1.6099 | Regression_eta: 0.5617 | Regression_pt: 1.7031 | Regression_sin_phi: 0.4662 | LR: 5.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0782 | max_loss: 0.0034 | max_backward: 0.2389 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.4007 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 4797.4 | uss_MB: 4038.4 | child_rss_MB: 4001.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:20:37,152] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:21:17,469] INFO : Step 25400/50000 rank0 | Train Loss: 7.5521 | Train Loss Unweighted: 2.8209 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5299 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1296 | Regression_eta: 0.0091 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8202 | Classification_binary: 2.5299 | Regression_cos_phi: 0.4246 | Regression_energy: 1.4196 | Regression_eta: 0.4462 | Regression_pt: 1.4572 | Regression_sin_phi: 0.4545 | LR: 5.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6577s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0413 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3107 | max_optimizer_step: 0.1362 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4797.4 | uss_MB: 4038.6 | child_rss_MB: 4026.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:21:17,682] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:21:58,893] INFO : Step 25500/50000 rank0 | Train Loss: 8.2759 | Train Loss Unweighted: 2.8810 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5547 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1397 | Regression_eta: 0.0134 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9174 | Classification_binary: 2.5547 | Regression_cos_phi: 0.5369 | Regression_energy: 1.5307 | Regression_eta: 0.6612 | Regression_pt: 1.6713 | Regression_sin_phi: 0.4038 | LR: 5.36e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6767s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1824 | max_loss: 0.0037 | max_backward: 0.2039 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 4004.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:21:59,128] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:22:41,407] INFO : Step 25600/50000 rank0 | Train Loss: 7.4296 | Train Loss Unweighted: 2.7492 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4720 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1208 | Regression_eta: 0.0110 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8064 | Classification_binary: 2.4720 | Regression_cos_phi: 0.5656 | Regression_energy: 1.3237 | Regression_eta: 0.5393 | Regression_pt: 1.3721 | Regression_sin_phi: 0.3504 | LR: 5.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.3106 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 4001.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:22:41,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:23:22,466] INFO : Step 25700/50000 rank0 | Train Loss: 8.5130 | Train Loss Unweighted: 2.8451 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5224 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1384 | Regression_eta: 0.0230 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8254 | Classification_binary: 2.5224 | Regression_cos_phi: 0.5394 | Regression_energy: 1.5165 | Regression_eta: 1.1292 | Regression_pt: 1.5481 | Regression_sin_phi: 0.4321 | LR: 5.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6909s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1087 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.3079 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 4026.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:23:22,681] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:03,306] INFO : Step 25800/50000 rank0 | Train Loss: 7.6826 | Train Loss Unweighted: 2.8629 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5724 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1258 | Regression_eta: 0.0107 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8090 | Classification_binary: 2.5724 | Regression_cos_phi: 0.4588 | Regression_energy: 1.3784 | Regression_eta: 0.5268 | Regression_pt: 1.4719 | Regression_sin_phi: 0.4654 | LR: 5.27e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6774s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.3069 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 3999.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:24:03,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:44,067] INFO : Step 25900/50000 rank0 | Train Loss: 8.3263 | Train Loss Unweighted: 2.9525 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.6273 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1417 | Regression_eta: 0.0109 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8502 | Classification_binary: 2.6273 | Regression_cos_phi: 0.6242 | Regression_energy: 1.5529 | Regression_eta: 0.5376 | Regression_pt: 1.6518 | Regression_sin_phi: 0.4823 | LR: 5.25e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5964s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3098 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4797.2 | uss_MB: 4038.5 | child_rss_MB: 4025.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:24:44,280] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:25:25,756] INFO : Step 26000/50000 rank0 | Train Loss: 7.7888 | Train Loss Unweighted: 2.7574 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4545 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1335 | Regression_eta: 0.0106 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9585 | Classification_binary: 2.4545 | Regression_cos_phi: 0.4818 | Regression_energy: 1.4629 | Regression_eta: 0.5222 | Regression_pt: 1.4957 | Regression_sin_phi: 0.4131 | LR: 5.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6045s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0890 | max_backward: 0.2030 | max_optimizer_step: 0.1280 | max_tensorboard_logging: 0.3376 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4797.2 | uss_MB: 4038.5 | child_rss_MB: 4005.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:25:25,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:25:26,004] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 14:25:26,181] INFO : Running validation on rank0 +rank0 [2026-10-28 14:25:26,181] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:25:26,181] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:25:27,567] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 14:25:27,712] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:25:27,845] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:25:27,919] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:25:27,988] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:25:28,057] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:25:28,126] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:25:28,197] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:25:28,310] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:25:28,443] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:25:28,574] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:25:28,645] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:25:28,715] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:25:28,784] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:25:28,854] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:25:28,925] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:25:29,043] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:25:29,050] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:25:29,054] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:25:29,058] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:25:29,062] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:25:29,065] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:25:29,069] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:25:29,073] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:25:29,074] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:25:29,078] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:25:29,081] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:25:29,085] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:25:29,086] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:25:29,090] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:25:29,094] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:25:29,098] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:25:36,794] INFO : VALIDATION | Step=26000/50000 | Train Loss=7.7888 | Valid Loss=7.8593 | Train Loss Unweighted=2.7574 | Valid Loss Unweighted=2.8014 | Stale=0 | ETA=178.2m +rank0 [2026-10-28 14:25:36,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:25:36,878] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:25:36,881] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:25:37,045] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:25:39,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:25:39,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:25:40,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:25:41,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:25:41,450] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:25:41,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:25:41,518] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:25:41,520] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:25:41,610] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:25:43,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:25:44,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:25:44,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:25:45,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:25:45,666] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:25:45,669] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:25:45,748] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:25:45,751] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:25:45,827] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:25:47,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:25:48,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:25:49,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:25:49,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:25:49,830] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:25:49,838] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:25:50,098] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:25:50,506] INFO : Plotted number of elements +rank0 [2026-10-28 14:25:51,951] INFO : Plotted jets +rank0 [2026-10-28 14:25:53,040] INFO : Plotted jet ratio +rank0 [2026-10-28 14:25:53,076] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:25:53,305] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:25:53,692] INFO : Plotted number of elements +rank0 [2026-10-28 14:25:55,653] INFO : Plotted jets +rank0 [2026-10-28 14:25:56,773] INFO : Plotted jet ratio +rank0 [2026-10-28 14:25:56,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 14:25:57,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 14:25:57,381] INFO : Plotted number of elements +rank0 [2026-10-28 14:25:58,762] INFO : Plotted jets +rank0 [2026-10-28 14:25:59,835] INFO : Plotted jet ratio +rank0 [2026-10-28 14:26:40,891] INFO : Step 26100/50000 rank0 | Train Loss: 7.8285 | Train Loss Unweighted: 2.8849 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5815 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1317 | Regression_eta: 0.0106 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8372 | Classification_binary: 2.5815 | Regression_cos_phi: 0.4195 | Regression_energy: 1.4431 | Regression_eta: 0.5228 | Regression_pt: 1.5550 | Regression_sin_phi: 0.4695 | LR: 5.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6251s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0798 | max_backward: 0.2023 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3196 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4985.1 | uss_MB: 4227.7 | child_rss_MB: 4005.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:26:41,104] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:27:23,306] INFO : Step 26200/50000 rank0 | Train Loss: 7.6889 | Train Loss Unweighted: 2.7059 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3885 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1403 | Regression_eta: 0.0089 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7751 | Classification_binary: 2.3885 | Regression_cos_phi: 0.4902 | Regression_energy: 1.5377 | Regression_eta: 0.4383 | Regression_pt: 1.6573 | Regression_sin_phi: 0.4018 | LR: 5.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7833s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0019 | max_forward: 0.1578 | max_loss: 0.0034 | max_backward: 0.3937 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0138 | Memory: rss_MB: 4985.1 | uss_MB: 4229.8 | child_rss_MB: 4030.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:27:23,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:28:04,576] INFO : Step 26300/50000 rank0 | Train Loss: 8.3302 | Train Loss Unweighted: 2.8500 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5154 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1472 | Regression_eta: 0.0148 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8001 | Classification_binary: 2.5154 | Regression_cos_phi: 0.5347 | Regression_energy: 1.6124 | Regression_eta: 0.7280 | Regression_pt: 1.6853 | Regression_sin_phi: 0.4541 | LR: 5.13e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6766s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1270 | max_loss: 0.0032 | max_backward: 0.2025 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.3118 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4870.9 | uss_MB: 4111.3 | child_rss_MB: 3996.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:28:04,790] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:28:45,553] INFO : Step 26400/50000 rank0 | Train Loss: 7.3076 | Train Loss Unweighted: 2.7554 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4706 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1241 | Regression_eta: 0.0088 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8672 | Classification_binary: 2.4706 | Regression_cos_phi: 0.3752 | Regression_energy: 1.3601 | Regression_eta: 0.4313 | Regression_pt: 1.4707 | Regression_sin_phi: 0.3325 | LR: 5.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7017s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3056 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3864 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4872.9 | uss_MB: 4112.6 | child_rss_MB: 3999.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:28:45,767] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:29:26,914] INFO : Step 26500/50000 rank0 | Train Loss: 8.3463 | Train Loss Unweighted: 2.8418 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5301 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1325 | Regression_eta: 0.0209 | Regression_pt: 0.1255 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8170 | Classification_binary: 2.5301 | Regression_cos_phi: 0.5296 | Regression_energy: 1.4513 | Regression_eta: 1.0254 | Regression_pt: 1.5048 | Regression_sin_phi: 0.4881 | LR: 5.08e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8906s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0831 | max_forward: 0.1044 | max_loss: 0.0033 | max_backward: 0.3932 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.3809 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4872.9 | uss_MB: 4114.9 | child_rss_MB: 3995.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:29:27,128] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:07,343] INFO : Step 26600/50000 rank0 | Train Loss: 8.0200 | Train Loss Unweighted: 2.8519 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5497 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1307 | Regression_eta: 0.0135 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8470 | Classification_binary: 2.5497 | Regression_cos_phi: 0.5643 | Regression_energy: 1.4316 | Regression_eta: 0.6647 | Regression_pt: 1.4899 | Regression_sin_phi: 0.4728 | LR: 5.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6870s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0646 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3111 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.2993 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4872.9 | uss_MB: 4112.8 | child_rss_MB: 4027.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:07,554] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:49,941] INFO : Step 26700/50000 rank0 | Train Loss: 8.8658 | Train Loss Unweighted: 2.9904 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6410 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1532 | Regression_eta: 0.0182 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9218 | Classification_binary: 2.6410 | Regression_cos_phi: 0.5631 | Regression_energy: 1.6784 | Regression_eta: 0.8952 | Regression_pt: 1.7145 | Regression_sin_phi: 0.4518 | LR: 5.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5895s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0015 | max_forward: 0.1833 | max_loss: 0.0035 | max_backward: 0.2025 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4872.9 | uss_MB: 4112.9 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:50,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:31:30,390] INFO : Step 26800/50000 rank0 | Train Loss: 7.2546 | Train Loss Unweighted: 2.7137 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.4331 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1241 | Regression_eta: 0.0099 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7953 | Classification_binary: 2.4331 | Regression_cos_phi: 0.3894 | Regression_energy: 1.3595 | Regression_eta: 0.4865 | Regression_pt: 1.4156 | Regression_sin_phi: 0.3751 | LR: 4.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6436s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.2627 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3979 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4872.9 | uss_MB: 4112.9 | child_rss_MB: 3999.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:31:30,602] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:32:12,105] INFO : Step 26900/50000 rank0 | Train Loss: 8.4342 | Train Loss Unweighted: 2.8619 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5072 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1556 | Regression_eta: 0.0153 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9041 | Classification_binary: 2.5072 | Regression_cos_phi: 0.3900 | Regression_energy: 1.7050 | Regression_eta: 0.7519 | Regression_pt: 1.8410 | Regression_sin_phi: 0.3351 | LR: 4.96e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5830s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0961 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4870.7 | uss_MB: 4111.2 | child_rss_MB: 4000.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:32:12,337] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:53,950] INFO : Step 27000/50000 rank0 | Train Loss: 7.7063 | Train Loss Unweighted: 2.7858 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4860 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1306 | Regression_eta: 0.0095 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8215 | Classification_binary: 2.4860 | Regression_cos_phi: 0.5685 | Regression_energy: 1.4305 | Regression_eta: 0.4660 | Regression_pt: 1.5290 | Regression_sin_phi: 0.4049 | LR: 4.94e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2282 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4870.7 | uss_MB: 4112.5 | child_rss_MB: 3998.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:32:54,164] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:54,260] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 14:32:54,434] INFO : Running validation on rank0 +rank0 [2026-10-28 14:32:54,435] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:32:54,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:32:55,757] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 14:32:55,904] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:32:56,037] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:32:56,110] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:32:56,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:32:56,249] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:32:56,320] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:32:56,389] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:32:56,502] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:32:56,635] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:32:56,767] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:32:56,838] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:32:56,908] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:32:56,977] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:32:57,047] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:32:57,117] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:32:57,232] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:32:57,238] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:32:57,241] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:32:57,242] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:32:57,246] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:32:57,248] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:32:57,249] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:32:57,252] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:32:57,255] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:32:57,258] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:32:57,259] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:32:57,262] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:32:57,263] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:32:57,265] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:32:57,266] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:32:57,269] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:33:04,897] INFO : VALIDATION | Step=27000/50000 | Train Loss=7.7063 | Valid Loss=7.9091 | Train Loss Unweighted=2.7858 | Valid Loss Unweighted=2.7975 | Stale=1 | ETA=170.8m +rank0 [2026-10-28 14:33:04,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:04,975] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:33:04,978] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:33:05,145] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:33:07,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:33:07,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:33:08,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:33:08,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:33:09,138] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:33:09,142] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:09,209] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:33:09,212] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:33:09,304] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:33:11,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:33:11,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:33:12,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:33:13,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:33:13,223] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:33:13,226] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:13,290] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:33:13,293] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:33:13,386] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:33:15,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:33:15,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:33:16,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:33:17,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:33:17,361] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:33:17,369] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:33:17,624] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:33:18,026] INFO : Plotted number of elements +rank0 [2026-10-28 14:33:19,468] INFO : Plotted jets +rank0 [2026-10-28 14:33:21,079] INFO : Plotted jet ratio +rank0 [2026-10-28 14:33:21,112] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:33:21,348] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:33:21,739] INFO : Plotted number of elements +rank0 [2026-10-28 14:33:23,146] INFO : Plotted jets +rank0 [2026-10-28 14:33:24,268] INFO : Plotted jet ratio +rank0 [2026-10-28 14:33:24,301] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 14:33:24,506] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 14:33:24,872] INFO : Plotted number of elements +rank0 [2026-10-28 14:33:26,256] INFO : Plotted jets +rank0 [2026-10-28 14:33:27,311] INFO : Plotted jet ratio +rank0 [2026-10-28 14:34:09,011] INFO : Step 27100/50000 rank0 | Train Loss: 7.6648 | Train Loss Unweighted: 2.7359 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4406 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1276 | Regression_eta: 0.0106 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8415 | Classification_binary: 2.4406 | Regression_cos_phi: 0.4980 | Regression_energy: 1.3984 | Regression_eta: 0.5235 | Regression_pt: 1.4916 | Regression_sin_phi: 0.4712 | LR: 4.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7228s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3058 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3896 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5060.2 | uss_MB: 4302.0 | child_rss_MB: 3999.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:09,222] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:50,511] INFO : Step 27200/50000 rank0 | Train Loss: 7.5980 | Train Loss Unweighted: 2.7749 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4839 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1261 | Regression_eta: 0.0106 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7811 | Classification_binary: 2.4839 | Regression_cos_phi: 0.5024 | Regression_energy: 1.3815 | Regression_eta: 0.5195 | Regression_pt: 1.4779 | Regression_sin_phi: 0.4519 | LR: 4.88e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.5562s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0263 | max_backward: 0.2027 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3213 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5060.2 | uss_MB: 4301.9 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:50,723] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:35:31,204] INFO : Step 27300/50000 rank0 | Train Loss: 8.0099 | Train Loss Unweighted: 2.9050 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5891 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1397 | Regression_eta: 0.0153 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8067 | Classification_binary: 2.5891 | Regression_cos_phi: 0.4164 | Regression_energy: 1.5309 | Regression_eta: 0.7536 | Regression_pt: 1.5881 | Regression_sin_phi: 0.3251 | LR: 4.85e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5637s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0197 | max_backward: 0.2025 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3480 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4942.4 | uss_MB: 4184.0 | child_rss_MB: 4001.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:35:31,417] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:36:13,292] INFO : Step 27400/50000 rank0 | Train Loss: 7.5353 | Train Loss Unweighted: 2.7477 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4606 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1243 | Regression_eta: 0.0097 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8770 | Classification_binary: 2.4606 | Regression_cos_phi: 0.5107 | Regression_energy: 1.3616 | Regression_eta: 0.4754 | Regression_pt: 1.4476 | Regression_sin_phi: 0.4025 | LR: 4.82e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6135s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2118 | max_optimizer_step: 0.1369 | max_tensorboard_logging: 0.3793 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4944.4 | uss_MB: 4185.3 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:36:13,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:36:54,273] INFO : Step 27500/50000 rank0 | Train Loss: 8.0608 | Train Loss Unweighted: 2.8422 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5451 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1261 | Regression_eta: 0.0141 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8556 | Classification_binary: 2.5451 | Regression_cos_phi: 0.6163 | Regression_energy: 1.3819 | Regression_eta: 0.6950 | Regression_pt: 1.4595 | Regression_sin_phi: 0.5074 | LR: 4.80e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6556s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1605 | max_loss: 0.0037 | max_backward: 0.2025 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:36:54,503] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:37:35,038] INFO : Step 27600/50000 rank0 | Train Loss: 7.2782 | Train Loss Unweighted: 2.7160 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4367 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1214 | Regression_eta: 0.0076 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8747 | Classification_binary: 2.4367 | Regression_cos_phi: 0.4478 | Regression_energy: 1.3298 | Regression_eta: 0.3758 | Regression_pt: 1.4266 | Regression_sin_phi: 0.3869 | LR: 4.77e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6995s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1405 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.3016 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3998.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:37:35,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:38:15,694] INFO : Step 27700/50000 rank0 | Train Loss: 7.4335 | Train Loss Unweighted: 2.6426 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3520 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1258 | Regression_eta: 0.0099 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8414 | Classification_binary: 2.3520 | Regression_cos_phi: 0.4946 | Regression_energy: 1.3786 | Regression_eta: 0.4844 | Regression_pt: 1.4792 | Regression_sin_phi: 0.4033 | LR: 4.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6010s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0927 | max_loss: 0.0034 | max_backward: 0.2023 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3995.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:38:15,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:38:56,886] INFO : Step 27800/50000 rank0 | Train Loss: 7.8478 | Train Loss Unweighted: 2.8224 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5055 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1384 | Regression_eta: 0.0102 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8466 | Classification_binary: 2.5055 | Regression_cos_phi: 0.4231 | Regression_energy: 1.5162 | Regression_eta: 0.5003 | Regression_pt: 1.6503 | Regression_sin_phi: 0.4057 | LR: 4.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5659s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0031 | max_backward: 0.2143 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3997.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:38:57,098] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:39:38,169] INFO : Step 27900/50000 rank0 | Train Loss: 7.1909 | Train Loss Unweighted: 2.6822 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4104 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1211 | Regression_eta: 0.0084 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8345 | Classification_binary: 2.4104 | Regression_cos_phi: 0.4672 | Regression_energy: 1.3270 | Regression_eta: 0.4142 | Regression_pt: 1.3347 | Regression_sin_phi: 0.4029 | LR: 4.68e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8285s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0252 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.1390 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4942.9 | uss_MB: 4183.9 | child_rss_MB: 3995.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:39:38,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:40:19,331] INFO : Step 28000/50000 rank0 | Train Loss: 7.8859 | Train Loss Unweighted: 2.7710 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4478 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1435 | Regression_eta: 0.0100 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8516 | Classification_binary: 2.4478 | Regression_cos_phi: 0.4599 | Regression_energy: 1.5718 | Regression_eta: 0.4939 | Regression_pt: 1.6616 | Regression_sin_phi: 0.3992 | LR: 4.66e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.6439s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0491 | max_forward: 0.0775 | max_loss: 0.0029 | max_backward: 0.3096 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4942.9 | uss_MB: 4185.1 | child_rss_MB: 4015.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:40:19,544] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:40:19,564] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 14:40:19,737] INFO : Running validation on rank0 +rank0 [2026-10-28 14:40:19,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:40:19,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:40:21,082] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 14:40:21,227] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:40:21,358] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:40:21,433] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:40:21,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:40:21,571] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:40:21,641] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:40:21,710] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:40:21,822] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:40:21,955] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:40:22,086] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:40:22,158] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:40:22,227] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:40:22,295] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:40:22,364] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:40:22,434] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:40:22,530] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:40:22,537] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:40:22,540] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:40:22,543] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:40:22,543] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:40:22,546] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:40:22,549] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:40:22,550] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:40:22,553] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:40:22,554] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:40:22,557] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:40:22,559] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:40:22,561] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:40:22,563] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:40:22,566] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:40:22,567] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:40:30,131] INFO : VALIDATION | Step=28000/50000 | Train Loss=7.8859 | Valid Loss=7.8292 | Train Loss Unweighted=2.7710 | Valid Loss Unweighted=2.7918 | Stale=0 | ETA=163.4m +rank0 [2026-10-28 14:40:30,141] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:40:30,200] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:40:30,204] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:40:30,345] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:40:32,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:40:33,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:40:33,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:40:34,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:40:34,504] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:40:34,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:40:34,588] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:40:34,591] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:40:34,667] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:40:36,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:40:37,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:40:37,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:40:38,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:40:38,695] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:40:38,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:40:38,777] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:40:38,779] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:40:38,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:40:40,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:40:41,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:40:42,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:40:42,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:40:42,881] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:40:42,889] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:40:43,150] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:40:43,553] INFO : Plotted number of elements +rank0 [2026-10-28 14:40:44,996] INFO : Plotted jets +rank0 [2026-10-28 14:40:46,084] INFO : Plotted jet ratio +rank0 [2026-10-28 14:40:46,116] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:40:46,349] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:40:47,312] INFO : Plotted number of elements +rank0 [2026-10-28 14:40:48,706] INFO : Plotted jets +rank0 [2026-10-28 14:40:49,815] INFO : Plotted jet ratio +rank0 [2026-10-28 14:40:49,848] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 14:40:50,057] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 14:40:50,419] INFO : Plotted number of elements +rank0 [2026-10-28 14:40:51,802] INFO : Plotted jets +rank0 [2026-10-28 14:40:52,872] INFO : Plotted jet ratio +rank0 [2026-10-28 14:41:34,296] INFO : Step 28100/50000 rank0 | Train Loss: 7.7474 | Train Loss Unweighted: 2.7896 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4996 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1240 | Regression_eta: 0.0141 | Regression_pt: 0.1197 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8587 | Classification_binary: 2.4996 | Regression_cos_phi: 0.4168 | Regression_energy: 1.3589 | Regression_eta: 0.6908 | Regression_pt: 1.4357 | Regression_sin_phi: 0.4868 | LR: 4.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6986s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0790 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3098 | max_optimizer_step: 0.1372 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5147.9 | uss_MB: 4393.1 | child_rss_MB: 4015.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:41:34,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:42:16,002] INFO : Step 28200/50000 rank0 | Train Loss: 7.9917 | Train Loss Unweighted: 2.8371 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.5477 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1207 | Regression_eta: 0.0147 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9598 | Classification_binary: 2.5477 | Regression_cos_phi: 0.5665 | Regression_energy: 1.3225 | Regression_eta: 0.7203 | Regression_pt: 1.4150 | Regression_sin_phi: 0.4600 | LR: 4.60e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6791s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1044 | max_loss: 0.0037 | max_backward: 0.2934 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.3423 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5147.9 | uss_MB: 4395.3 | child_rss_MB: 3992.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:42:16,212] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:42:57,314] INFO : Step 28300/50000 rank0 | Train Loss: 8.2479 | Train Loss Unweighted: 2.8117 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4892 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1406 | Regression_eta: 0.0159 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8546 | Classification_binary: 2.4892 | Regression_cos_phi: 0.4985 | Regression_energy: 1.5403 | Regression_eta: 0.7802 | Regression_pt: 1.5910 | Regression_sin_phi: 0.4940 | LR: 4.57e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6661s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0773 | max_loss: 0.0036 | max_backward: 0.2436 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.4067 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5014.9 | uss_MB: 4259.3 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:42:57,527] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:43:39,333] INFO : Step 28400/50000 rank0 | Train Loss: 7.6846 | Train Loss Unweighted: 2.7839 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4764 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1347 | Regression_eta: 0.0097 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8111 | Classification_binary: 2.4764 | Regression_cos_phi: 0.4418 | Regression_energy: 1.4757 | Regression_eta: 0.4786 | Regression_pt: 1.5931 | Regression_sin_phi: 0.4078 | LR: 4.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6473s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.1029 | max_backward: 0.2031 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3175 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5015.9 | uss_MB: 4260.6 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:43:39,545] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:44:20,010] INFO : Step 28500/50000 rank0 | Train Loss: 8.2545 | Train Loss Unweighted: 2.8759 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.5617 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1335 | Regression_eta: 0.0177 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7759 | Classification_binary: 2.5617 | Regression_cos_phi: 0.5886 | Regression_energy: 1.4630 | Regression_eta: 0.8702 | Regression_pt: 1.5739 | Regression_sin_phi: 0.4213 | LR: 4.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5660s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0294 | max_backward: 0.2032 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3193 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5015.9 | uss_MB: 4260.8 | child_rss_MB: 3992.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:44:20,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:01,998] INFO : Step 28600/50000 rank0 | Train Loss: 7.7300 | Train Loss Unweighted: 2.8020 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.4999 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1327 | Regression_eta: 0.0111 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7887 | Classification_binary: 2.4999 | Regression_cos_phi: 0.4323 | Regression_energy: 1.4542 | Regression_eta: 0.5434 | Regression_pt: 1.5299 | Regression_sin_phi: 0.4815 | LR: 4.49e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6821s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2732 | max_optimizer_step: 0.1357 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5015.9 | uss_MB: 4260.9 | child_rss_MB: 3993.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:45:02,211] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:42,712] INFO : Step 28700/50000 rank0 | Train Loss: 7.6298 | Train Loss Unweighted: 2.7918 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5067 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1229 | Regression_eta: 0.0141 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8228 | Classification_binary: 2.5067 | Regression_cos_phi: 0.4431 | Regression_energy: 1.3462 | Regression_eta: 0.6931 | Regression_pt: 1.4116 | Regression_sin_phi: 0.4062 | LR: 4.46e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5528s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0119 | max_backward: 0.2030 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3174 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5015.9 | uss_MB: 4260.9 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:45:42,926] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:46:24,351] INFO : Step 28800/50000 rank0 | Train Loss: 8.1128 | Train Loss Unweighted: 2.8401 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5096 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1456 | Regression_eta: 0.0119 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8217 | Classification_binary: 2.5096 | Regression_cos_phi: 0.4667 | Regression_energy: 1.5950 | Regression_eta: 0.5830 | Regression_pt: 1.7010 | Regression_sin_phi: 0.4359 | LR: 4.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5927s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1860 | max_loss: 0.0035 | max_backward: 0.2025 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.2870 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5015.9 | uss_MB: 4260.9 | child_rss_MB: 3992.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:46:24,568] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:04,895] INFO : Step 28900/50000 rank0 | Train Loss: 8.0833 | Train Loss Unweighted: 2.9385 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.6158 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1433 | Regression_eta: 0.0109 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7860 | Classification_binary: 2.6158 | Regression_cos_phi: 0.4850 | Regression_energy: 1.5702 | Regression_eta: 0.5339 | Regression_pt: 1.6506 | Regression_sin_phi: 0.4417 | LR: 4.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6157s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.2217 | max_optimizer_step: 0.1333 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5015.6 | uss_MB: 4259.4 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:47:05,109] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:45,887] INFO : Step 29000/50000 rank0 | Train Loss: 7.6657 | Train Loss Unweighted: 2.7995 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4936 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1345 | Regression_eta: 0.0081 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8545 | Classification_binary: 2.4936 | Regression_cos_phi: 0.4704 | Regression_energy: 1.4741 | Regression_eta: 0.3978 | Regression_pt: 1.5831 | Regression_sin_phi: 0.3921 | LR: 4.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6983s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1040 | max_loss: 0.0034 | max_backward: 0.3892 | max_optimizer_step: 0.1319 | max_tensorboard_logging: 0.2842 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5016.6 | uss_MB: 4262.8 | child_rss_MB: 4018.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:47:46,100] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:46,120] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 14:47:46,294] INFO : Running validation on rank0 +rank0 [2026-10-28 14:47:46,294] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:47:46,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:47:47,554] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 14:47:47,701] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:47:47,834] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:47:47,909] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:47:47,979] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:47:48,047] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:47:48,118] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:47:48,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:47:48,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:47:48,432] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:47:48,566] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:47:48,637] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:47:48,706] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:47:48,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:47:48,847] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:47:48,916] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:47:49,021] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:47:49,028] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:47:49,032] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:47:49,035] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:47:49,037] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:47:49,040] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:47:49,041] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:47:49,044] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:47:49,047] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:47:49,048] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:47:49,051] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:47:49,054] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:47:49,055] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:47:49,058] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:47:49,060] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:47:49,062] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:47:56,650] INFO : VALIDATION | Step=29000/50000 | Train Loss=7.6657 | Valid Loss=7.7437 | Train Loss Unweighted=2.7995 | Valid Loss Unweighted=2.7686 | Stale=0 | ETA=156.0m +rank0 [2026-10-28 14:47:56,659] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:47:56,739] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:47:56,742] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:47:56,938] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:47:59,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:47:59,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:48:00,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:48:01,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:48:01,341] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:48:01,344] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:48:01,400] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:48:01,403] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:48:01,516] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:48:03,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:48:03,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:48:04,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:48:05,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:48:05,378] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:48:05,381] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:48:05,435] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:48:05,438] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:48:05,542] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:48:07,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:48:08,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:48:08,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:48:09,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:48:09,574] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:48:09,598] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:48:09,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:48:10,261] INFO : Plotted number of elements +rank0 [2026-10-28 14:48:11,699] INFO : Plotted jets +rank0 [2026-10-28 14:48:12,792] INFO : Plotted jet ratio +rank0 [2026-10-28 14:48:12,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:48:13,072] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:48:14,011] INFO : Plotted number of elements +rank0 [2026-10-28 14:48:15,401] INFO : Plotted jets +rank0 [2026-10-28 14:48:16,512] INFO : Plotted jet ratio +rank0 [2026-10-28 14:48:16,546] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 14:48:16,751] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 14:48:17,112] INFO : Plotted number of elements +rank0 [2026-10-28 14:48:18,486] INFO : Plotted jets +rank0 [2026-10-28 14:48:19,551] INFO : Plotted jet ratio +rank0 [2026-10-28 14:49:00,522] INFO : Step 29100/50000 rank0 | Train Loss: 7.2658 | Train Loss Unweighted: 2.6978 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4216 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1199 | Regression_eta: 0.0095 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7767 | Classification_binary: 2.4216 | Regression_cos_phi: 0.4540 | Regression_energy: 1.3133 | Regression_eta: 0.4662 | Regression_pt: 1.4012 | Regression_sin_phi: 0.4329 | LR: 4.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7259s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0031 | max_backward: 0.3076 | max_optimizer_step: 0.1370 | max_tensorboard_logging: 0.3836 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5209.6 | uss_MB: 4454.7 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:49:00,734] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:49:42,613] INFO : Step 29200/50000 rank0 | Train Loss: 7.7446 | Train Loss Unweighted: 2.7666 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4489 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1413 | Regression_eta: 0.0095 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8346 | Classification_binary: 2.4489 | Regression_cos_phi: 0.4445 | Regression_energy: 1.5483 | Regression_eta: 0.4659 | Regression_pt: 1.6408 | Regression_sin_phi: 0.3616 | LR: 4.33e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6704s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2497 | max_optimizer_step: 0.1362 | max_tensorboard_logging: 0.3845 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5209.6 | uss_MB: 4454.7 | child_rss_MB: 3998.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:49:42,827] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:50:23,505] INFO : Step 29300/50000 rank0 | Train Loss: 7.7195 | Train Loss Unweighted: 2.8341 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5276 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1338 | Regression_eta: 0.0098 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8547 | Classification_binary: 2.5276 | Regression_cos_phi: 0.4052 | Regression_energy: 1.4659 | Regression_eta: 0.4829 | Regression_pt: 1.5882 | Regression_sin_phi: 0.3949 | LR: 4.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6471s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1778 | max_loss: 0.0031 | max_backward: 0.2021 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3126 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5087.9 | uss_MB: 4332.7 | child_rss_MB: 3995.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:50:23,729] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:04,597] INFO : Step 29400/50000 rank0 | Train Loss: 7.6670 | Train Loss Unweighted: 2.7807 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4831 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1284 | Regression_eta: 0.0074 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8691 | Classification_binary: 2.4831 | Regression_cos_phi: 0.5660 | Regression_energy: 1.4072 | Regression_eta: 0.3653 | Regression_pt: 1.5348 | Regression_sin_phi: 0.4416 | LR: 4.27e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6726s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.2764 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5088.9 | uss_MB: 4333.8 | child_rss_MB: 3999.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:51:04,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:45,925] INFO : Step 29500/50000 rank0 | Train Loss: 7.5591 | Train Loss Unweighted: 2.6428 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3435 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1301 | Regression_eta: 0.0085 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8622 | Classification_binary: 2.3435 | Regression_cos_phi: 0.4892 | Regression_energy: 1.4250 | Regression_eta: 0.4197 | Regression_pt: 1.5285 | Regression_sin_phi: 0.4911 | LR: 4.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7111s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3081 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3793 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5088.9 | uss_MB: 4333.9 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:51:46,138] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:52:26,978] INFO : Step 29600/50000 rank0 | Train Loss: 7.5619 | Train Loss Unweighted: 2.7819 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4856 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1296 | Regression_eta: 0.0091 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8423 | Classification_binary: 2.4856 | Regression_cos_phi: 0.4602 | Regression_energy: 1.4199 | Regression_eta: 0.4463 | Regression_pt: 1.5197 | Regression_sin_phi: 0.3879 | LR: 4.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7920s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1840 | max_loss: 0.0031 | max_backward: 0.2021 | max_optimizer_step: 0.2401 | max_tensorboard_logging: 0.3326 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 5088.9 | uss_MB: 4334.0 | child_rss_MB: 4004.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:52:27,191] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:08,715] INFO : Step 29700/50000 rank0 | Train Loss: 8.3498 | Train Loss Unweighted: 2.9355 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5898 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1539 | Regression_eta: 0.0109 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8089 | Classification_binary: 2.5898 | Regression_cos_phi: 0.5355 | Regression_energy: 1.6863 | Regression_eta: 0.5356 | Regression_pt: 1.7930 | Regression_sin_phi: 0.4007 | LR: 4.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7444s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1128 | max_loss: 0.0033 | max_backward: 0.3950 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.2875 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5088.9 | uss_MB: 4336.4 | child_rss_MB: 4026.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:53:08,927] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:49,620] INFO : Step 29800/50000 rank0 | Train Loss: 7.0682 | Train Loss Unweighted: 2.7709 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4958 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1211 | Regression_eta: 0.0069 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8180 | Classification_binary: 2.4958 | Regression_cos_phi: 0.3288 | Regression_energy: 1.3273 | Regression_eta: 0.3398 | Regression_pt: 1.4336 | Regression_sin_phi: 0.3248 | LR: 4.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6100s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2294 | max_optimizer_step: 0.1314 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5088.9 | uss_MB: 4334.4 | child_rss_MB: 3998.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:53:49,832] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:54:30,991] INFO : Step 29900/50000 rank0 | Train Loss: 8.6388 | Train Loss Unweighted: 2.8845 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5485 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1459 | Regression_eta: 0.0190 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8066 | Classification_binary: 2.5485 | Regression_cos_phi: 0.5477 | Regression_energy: 1.5987 | Regression_eta: 0.9336 | Regression_pt: 1.6439 | Regression_sin_phi: 0.5599 | LR: 4.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7372s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1038 | max_loss: 0.0032 | max_backward: 0.3595 | max_optimizer_step: 0.1318 | max_tensorboard_logging: 0.3413 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5087.7 | uss_MB: 4334.8 | child_rss_MB: 3997.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:54:31,206] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:11,921] INFO : Step 30000/50000 rank0 | Train Loss: 7.9552 | Train Loss Unweighted: 2.7797 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4734 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1334 | Regression_eta: 0.0127 | Regression_pt: 0.1266 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8158 | Classification_binary: 2.4734 | Regression_cos_phi: 0.4973 | Regression_energy: 1.4614 | Regression_eta: 0.6238 | Regression_pt: 1.5187 | Regression_sin_phi: 0.5648 | LR: 4.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5875s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0954 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.2021 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.7 | uss_MB: 4333.9 | child_rss_MB: 3998.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:55:12,143] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:12,161] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 14:55:12,336] INFO : Running validation on rank0 +rank0 [2026-10-28 14:55:12,336] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:55:12,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:55:13,652] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 14:55:13,798] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:55:13,929] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:55:14,003] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:55:14,073] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:55:14,142] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:55:14,212] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:55:14,281] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:55:14,394] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:55:14,525] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:55:14,656] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:55:14,727] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:55:14,797] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:55:14,865] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:55:14,935] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:55:15,004] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:55:15,121] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:55:15,128] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:55:15,131] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:55:15,134] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:55:15,136] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:55:15,138] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:55:15,141] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:55:15,143] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:55:15,144] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:55:15,148] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:55:15,150] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:55:15,153] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:55:15,157] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:55:15,159] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:55:15,160] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:55:15,163] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:55:22,696] INFO : VALIDATION | Step=30000/50000 | Train Loss=7.9552 | Valid Loss=7.7615 | Train Loss Unweighted=2.7797 | Valid Loss Unweighted=2.7671 | Stale=1 | ETA=148.5m +rank0 [2026-10-28 14:55:22,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:55:22,772] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:55:22,776] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:55:22,946] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:55:24,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:55:25,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:55:26,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:55:26,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:55:26,801] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:55:26,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:55:26,875] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:55:26,878] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:55:26,968] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:55:28,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:55:29,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:55:30,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:55:30,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:55:30,851] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:55:30,855] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:55:30,910] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:55:30,912] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:55:31,018] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:55:32,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:55:33,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:55:34,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:55:34,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:55:35,015] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:55:35,022] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:55:35,281] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:55:35,688] INFO : Plotted number of elements +rank0 [2026-10-28 14:55:37,127] INFO : Plotted jets +rank0 [2026-10-28 14:55:38,774] INFO : Plotted jet ratio +rank0 [2026-10-28 14:55:38,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:55:39,038] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:55:39,429] INFO : Plotted number of elements +rank0 [2026-10-28 14:55:40,831] INFO : Plotted jets +rank0 [2026-10-28 14:55:41,951] INFO : Plotted jet ratio +rank0 [2026-10-28 14:55:41,985] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 14:55:42,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 14:55:42,559] INFO : Plotted number of elements +rank0 [2026-10-28 14:55:43,943] INFO : Plotted jets +rank0 [2026-10-28 14:55:45,005] INFO : Plotted jet ratio +rank0 [2026-10-28 14:56:27,215] INFO : Step 30100/50000 rank0 | Train Loss: 7.8092 | Train Loss Unweighted: 2.7948 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4808 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1379 | Regression_eta: 0.0099 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8896 | Classification_binary: 2.4808 | Regression_cos_phi: 0.5110 | Regression_energy: 1.5115 | Regression_eta: 0.4864 | Regression_pt: 1.6163 | Regression_sin_phi: 0.3136 | LR: 4.08e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5735s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0154 | max_backward: 0.2027 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3146 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5279.3 | uss_MB: 4523.1 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:56:27,426] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:57:07,554] INFO : Step 30200/50000 rank0 | Train Loss: 7.8516 | Train Loss Unweighted: 2.7367 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4411 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1283 | Regression_eta: 0.0186 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8072 | Classification_binary: 2.4411 | Regression_cos_phi: 0.4632 | Regression_energy: 1.4053 | Regression_eta: 0.9126 | Regression_pt: 1.4210 | Regression_sin_phi: 0.4011 | LR: 4.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6577s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0151 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3091 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5279.3 | uss_MB: 4523.1 | child_rss_MB: 4024.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:57:07,767] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:57:49,220] INFO : Step 30300/50000 rank0 | Train Loss: 8.0430 | Train Loss Unweighted: 2.8609 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5496 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1352 | Regression_eta: 0.0162 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8482 | Classification_binary: 2.5496 | Regression_cos_phi: 0.4510 | Regression_energy: 1.4809 | Regression_eta: 0.7975 | Regression_pt: 1.5530 | Regression_sin_phi: 0.3628 | LR: 4.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6752s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.2201 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3875 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5279.3 | uss_MB: 4523.1 | child_rss_MB: 3998.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:57:49,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:58:31,509] INFO : Step 30400/50000 rank0 | Train Loss: 8.2332 | Train Loss Unweighted: 2.7203 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4006 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1375 | Regression_eta: 0.0210 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8657 | Classification_binary: 2.4006 | Regression_cos_phi: 0.4740 | Regression_energy: 1.5070 | Regression_eta: 1.0340 | Regression_pt: 1.5518 | Regression_sin_phi: 0.4001 | LR: 4.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1497 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5159.8 | uss_MB: 4403.0 | child_rss_MB: 3998.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:58:31,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:12,545] INFO : Step 30500/50000 rank0 | Train Loss: 8.1211 | Train Loss Unweighted: 2.7258 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4250 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1255 | Regression_eta: 0.0227 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8319 | Classification_binary: 2.4250 | Regression_cos_phi: 0.4696 | Regression_energy: 1.3751 | Regression_eta: 1.1138 | Regression_pt: 1.4493 | Regression_sin_phi: 0.4565 | LR: 3.98e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7341s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5159.8 | uss_MB: 4403.0 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:59:12,758] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:54,647] INFO : Step 30600/50000 rank0 | Train Loss: 7.5510 | Train Loss Unweighted: 2.7837 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4941 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1253 | Regression_eta: 0.0114 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8610 | Classification_binary: 2.4941 | Regression_cos_phi: 0.4442 | Regression_energy: 1.3724 | Regression_eta: 0.5598 | Regression_pt: 1.4700 | Regression_sin_phi: 0.3494 | LR: 3.95e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0446 | max_backward: 0.2031 | max_optimizer_step: 0.1368 | max_tensorboard_logging: 0.3083 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5159.8 | uss_MB: 4403.2 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:59:54,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:00:35,697] INFO : Step 30700/50000 rank0 | Train Loss: 7.9790 | Train Loss Unweighted: 2.8122 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4980 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1392 | Regression_eta: 0.0142 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8294 | Classification_binary: 2.4980 | Regression_cos_phi: 0.4758 | Regression_energy: 1.5256 | Regression_eta: 0.6997 | Regression_pt: 1.5571 | Regression_sin_phi: 0.3935 | LR: 3.92e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6313s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0097 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3071 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5159.8 | uss_MB: 4403.2 | child_rss_MB: 4021.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:00:35,910] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:17,853] INFO : Step 30800/50000 rank0 | Train Loss: 7.3387 | Train Loss Unweighted: 2.7645 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4872 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1212 | Regression_eta: 0.0107 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8057 | Classification_binary: 2.4872 | Regression_cos_phi: 0.4280 | Regression_energy: 1.3285 | Regression_eta: 0.5240 | Regression_pt: 1.3916 | Regression_sin_phi: 0.3737 | LR: 3.90e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7085s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1843 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.2848 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5159.8 | uss_MB: 4403.4 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:01:18,082] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:59,346] INFO : Step 30900/50000 rank0 | Train Loss: 7.3086 | Train Loss Unweighted: 2.7052 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4285 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1193 | Regression_eta: 0.0099 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8570 | Classification_binary: 2.4285 | Regression_cos_phi: 0.3481 | Regression_energy: 1.3069 | Regression_eta: 0.4881 | Regression_pt: 1.3960 | Regression_sin_phi: 0.4840 | LR: 3.87e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7842s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1456 | max_loss: 0.0035 | max_backward: 0.3893 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.2928 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5160.8 | uss_MB: 4405.8 | child_rss_MB: 4026.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 15:01:59,558] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:02:39,670] INFO : Step 31000/50000 rank0 | Train Loss: 7.8594 | Train Loss Unweighted: 2.7765 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4650 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1337 | Regression_eta: 0.0103 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8798 | Classification_binary: 2.4650 | Regression_cos_phi: 0.4848 | Regression_energy: 1.4655 | Regression_eta: 0.5074 | Regression_pt: 1.6142 | Regression_sin_phi: 0.4427 | LR: 3.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5333s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0055 | max_backward: 0.2031 | max_optimizer_step: 0.1301 | max_tensorboard_logging: 0.3298 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5158.9 | uss_MB: 4402.9 | child_rss_MB: 3998.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:02:39,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:02:39,924] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 15:02:40,107] INFO : Running validation on rank0 +rank0 [2026-10-28 15:02:40,107] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:02:40,107] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:02:41,543] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 15:02:41,687] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:02:41,819] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:02:41,894] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:02:41,963] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:02:42,032] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:02:42,102] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:02:42,171] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:02:42,285] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:02:42,416] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:02:42,548] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:02:42,620] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:02:42,689] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:02:42,757] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:02:42,826] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:02:42,895] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:02:43,011] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:02:43,018] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:02:43,020] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:02:43,022] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:02:43,024] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:02:43,025] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:02:43,028] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:02:43,031] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:02:43,032] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:02:43,035] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:02:43,035] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:02:43,038] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:02:43,041] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:02:43,042] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:02:43,044] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:02:43,045] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:02:50,527] INFO : VALIDATION | Step=31000/50000 | Train Loss=7.8594 | Valid Loss=7.6697 | Train Loss Unweighted=2.7765 | Valid Loss Unweighted=2.7551 | Stale=0 | ETA=141.1m +rank0 [2026-10-28 15:02:50,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:02:50,637] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:02:50,641] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:02:50,810] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:02:52,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:02:53,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:02:54,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:02:54,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:02:54,770] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:02:54,773] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:02:54,836] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:02:54,839] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:02:54,939] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:02:56,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:02:57,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:02:58,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:02:58,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:02:58,878] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:02:58,881] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:02:58,945] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:02:58,948] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:02:59,052] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:03:00,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:03:01,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:03:02,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:03:02,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:03:03,011] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:03:03,019] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:03:03,278] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:03:03,682] INFO : Plotted number of elements +rank0 [2026-10-28 15:03:05,120] INFO : Plotted jets +rank0 [2026-10-28 15:03:06,205] INFO : Plotted jet ratio +rank0 [2026-10-28 15:03:06,238] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:03:06,467] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:03:06,857] INFO : Plotted number of elements +rank0 [2026-10-28 15:03:08,827] INFO : Plotted jets +rank0 [2026-10-28 15:03:09,932] INFO : Plotted jet ratio +rank0 [2026-10-28 15:03:09,965] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 15:03:10,178] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 15:03:10,540] INFO : Plotted number of elements +rank0 [2026-10-28 15:03:11,924] INFO : Plotted jets +rank0 [2026-10-28 15:03:12,988] INFO : Plotted jet ratio +rank0 [2026-10-28 15:03:55,360] INFO : Step 31100/50000 rank0 | Train Loss: 7.7979 | Train Loss Unweighted: 2.6861 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3795 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1342 | Regression_eta: 0.0154 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7933 | Classification_binary: 2.3795 | Regression_cos_phi: 0.5001 | Regression_energy: 1.4708 | Regression_eta: 0.7581 | Regression_pt: 1.5216 | Regression_sin_phi: 0.3745 | LR: 3.82e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6569s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0193 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3099 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.2790 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5355.2 | uss_MB: 4600.6 | child_rss_MB: 4030.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:03:55,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:04:37,224] INFO : Step 31200/50000 rank0 | Train Loss: 7.5580 | Train Loss Unweighted: 2.7084 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4108 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1312 | Regression_eta: 0.0103 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8069 | Classification_binary: 2.4108 | Regression_cos_phi: 0.4716 | Regression_energy: 1.4380 | Regression_eta: 0.5063 | Regression_pt: 1.5027 | Regression_sin_phi: 0.4218 | LR: 3.79e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7250s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0037 | max_backward: 0.3091 | max_optimizer_step: 0.1362 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5355.2 | uss_MB: 4600.6 | child_rss_MB: 4000.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:04:37,437] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 3% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:05:18,840] INFO : Step 31300/50000 rank0 | Train Loss: 7.7047 | Train Loss Unweighted: 2.6989 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3771 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1425 | Regression_eta: 0.0075 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8819 | Classification_binary: 2.3771 | Regression_cos_phi: 0.4567 | Regression_energy: 1.5615 | Regression_eta: 0.3682 | Regression_pt: 1.6834 | Regression_sin_phi: 0.3760 | LR: 3.76e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7907s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1050 | max_loss: 0.0035 | max_backward: 0.3724 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.3308 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5355.2 | uss_MB: 4602.7 | child_rss_MB: 3997.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 15:05:19,054] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:05:59,462] INFO : Step 31400/50000 rank0 | Train Loss: 7.8413 | Train Loss Unweighted: 2.7043 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3956 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1344 | Regression_eta: 0.0125 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7880 | Classification_binary: 2.3956 | Regression_cos_phi: 0.5036 | Regression_energy: 1.4730 | Regression_eta: 0.6154 | Regression_pt: 1.5530 | Regression_sin_phi: 0.5126 | LR: 3.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7431s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3102 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.4013 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 5235.6 | uss_MB: 4479.3 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:05:59,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:06:41,109] INFO : Step 31500/50000 rank0 | Train Loss: 8.0302 | Train Loss Unweighted: 2.8475 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5376 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1323 | Regression_eta: 0.0134 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8800 | Classification_binary: 2.5376 | Regression_cos_phi: 0.4667 | Regression_energy: 1.4493 | Regression_eta: 0.6566 | Regression_pt: 1.5743 | Regression_sin_phi: 0.4657 | LR: 3.71e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1788 | max_loss: 0.0035 | max_backward: 0.2028 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5235.6 | uss_MB: 4479.7 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:06:41,338] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:07:21,939] INFO : Step 31600/50000 rank0 | Train Loss: 7.8913 | Train Loss Unweighted: 2.8011 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4812 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1414 | Regression_eta: 0.0094 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 2.4812 | Regression_cos_phi: 0.5051 | Regression_energy: 1.5494 | Regression_eta: 0.4614 | Regression_pt: 1.6490 | Regression_sin_phi: 0.4317 | LR: 3.69e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6202s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0082 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3079 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2816 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5235.6 | uss_MB: 4479.7 | child_rss_MB: 4018.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:07:22,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:03,843] INFO : Step 31700/50000 rank0 | Train Loss: 8.3653 | Train Loss Unweighted: 2.9146 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5871 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1414 | Regression_eta: 0.0139 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8978 | Classification_binary: 2.5871 | Regression_cos_phi: 0.5229 | Regression_energy: 1.5490 | Regression_eta: 0.6821 | Regression_pt: 1.6544 | Regression_sin_phi: 0.4720 | LR: 3.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2350s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1004 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.3930 | max_optimizer_step: 0.1358 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5236.6 | uss_MB: 4482.3 | child_rss_MB: 3995.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:08:04,056] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:44,932] INFO : Step 31800/50000 rank0 | Train Loss: 7.4813 | Train Loss Unweighted: 2.7657 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4805 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1218 | Regression_eta: 0.0095 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7805 | Classification_binary: 2.4805 | Regression_cos_phi: 0.5004 | Regression_energy: 1.3343 | Regression_eta: 0.4659 | Regression_pt: 1.4748 | Regression_sin_phi: 0.4448 | LR: 3.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6539s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0515 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.3086 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.2859 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5237.6 | uss_MB: 4480.3 | child_rss_MB: 4016.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:08:45,146] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:09:25,950] INFO : Step 31900/50000 rank0 | Train Loss: 7.2305 | Train Loss Unweighted: 2.7003 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.4220 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1214 | Regression_eta: 0.0112 | Regression_pt: 0.1181 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7497 | Classification_binary: 2.4220 | Regression_cos_phi: 0.3491 | Regression_energy: 1.3300 | Regression_eta: 0.5501 | Regression_pt: 1.4162 | Regression_sin_phi: 0.4135 | LR: 3.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6956s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2791 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5237.6 | uss_MB: 4480.3 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:09:26,164] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:07,604] INFO : Step 32000/50000 rank0 | Train Loss: 7.6841 | Train Loss Unweighted: 2.7223 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4355 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1215 | Regression_eta: 0.0126 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7985 | Classification_binary: 2.4355 | Regression_cos_phi: 0.6335 | Regression_energy: 1.3311 | Regression_eta: 0.6219 | Regression_pt: 1.4361 | Regression_sin_phi: 0.4275 | LR: 3.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7240s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1842 | max_loss: 0.0035 | max_backward: 0.2026 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.2971 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5238.6 | uss_MB: 4479.2 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:10:07,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:07,839] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 15:10:08,018] INFO : Running validation on rank0 +rank0 [2026-10-28 15:10:08,019] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:10:08,019] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:10:09,356] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 15:10:09,503] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:10:09,637] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:10:09,711] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:10:09,780] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:10:09,848] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:10:09,918] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:10:09,987] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:10:10,100] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:10:10,233] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:10:10,365] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:10:10,437] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:10:10,507] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:10:10,575] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:10:10,645] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:10:10,715] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:10:10,811] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:10:10,818] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:10:10,821] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:10:10,824] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:10:10,825] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:10:10,828] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:10:10,831] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:10:10,832] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:10:10,835] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:10:10,838] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:10:10,841] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:10:10,844] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:10:10,845] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:10:10,847] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:10:10,848] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:10:10,852] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:10:18,413] INFO : VALIDATION | Step=32000/50000 | Train Loss=7.6841 | Valid Loss=7.6540 | Train Loss Unweighted=2.7223 | Valid Loss Unweighted=2.7312 | Stale=0 | ETA=133.7m +rank0 [2026-10-28 15:10:18,423] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:10:18,608] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:10:18,612] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:10:18,795] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:10:20,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:10:21,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:10:22,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:10:22,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:10:23,134] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:10:23,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:10:23,215] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:10:23,218] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:10:23,307] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:10:25,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:10:25,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:10:26,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:10:27,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:10:27,150] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:10:27,153] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:10:27,220] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:10:27,223] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:10:27,316] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:10:29,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:10:29,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:10:30,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:10:31,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:10:31,326] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:10:31,333] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:10:31,595] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:10:32,001] INFO : Plotted number of elements +rank0 [2026-10-28 15:10:33,435] INFO : Plotted jets +rank0 [2026-10-28 15:10:34,529] INFO : Plotted jet ratio +rank0 [2026-10-28 15:10:34,563] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:10:34,785] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:10:35,175] INFO : Plotted number of elements +rank0 [2026-10-28 15:10:37,152] INFO : Plotted jets +rank0 [2026-10-28 15:10:38,257] INFO : Plotted jet ratio +rank0 [2026-10-28 15:10:38,290] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 15:10:38,494] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 15:10:38,859] INFO : Plotted number of elements +rank0 [2026-10-28 15:10:40,237] INFO : Plotted jets +rank0 [2026-10-28 15:10:41,299] INFO : Plotted jet ratio +rank0 [2026-10-28 15:11:22,413] INFO : Step 32100/50000 rank0 | Train Loss: 7.6895 | Train Loss Unweighted: 2.7110 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3978 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1387 | Regression_eta: 0.0102 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8630 | Classification_binary: 2.3978 | Regression_cos_phi: 0.4430 | Regression_energy: 1.5198 | Regression_eta: 0.5021 | Regression_pt: 1.6033 | Regression_sin_phi: 0.3605 | LR: 3.56e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.7228s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1045 | max_loss: 0.0035 | max_backward: 0.3116 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3319 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5441.1 | uss_MB: 4684.6 | child_rss_MB: 3996.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 15:11:22,627] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:12:03,399] INFO : Step 32200/50000 rank0 | Train Loss: 8.2772 | Train Loss Unweighted: 2.8474 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5065 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1522 | Regression_eta: 0.0106 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8411 | Classification_binary: 2.5065 | Regression_cos_phi: 0.5735 | Regression_energy: 1.6680 | Regression_eta: 0.5191 | Regression_pt: 1.7398 | Regression_sin_phi: 0.4292 | LR: 3.53e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.7169s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.1838 | max_loss: 0.0036 | max_backward: 0.2026 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5441.1 | uss_MB: 4682.5 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:12:03,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:12:45,479] INFO : Step 32300/50000 rank0 | Train Loss: 7.6594 | Train Loss Unweighted: 2.7335 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4416 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1256 | Regression_eta: 0.0091 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9143 | Classification_binary: 2.4416 | Regression_cos_phi: 0.6323 | Regression_energy: 1.3759 | Regression_eta: 0.4497 | Regression_pt: 1.4696 | Regression_sin_phi: 0.3760 | LR: 3.51e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7571s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1757 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.2929 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5441.1 | uss_MB: 4682.6 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:12:45,695] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:13:26,660] INFO : Step 32400/50000 rank0 | Train Loss: 7.4844 | Train Loss Unweighted: 2.7397 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.4431 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1313 | Regression_eta: 0.0094 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8027 | Classification_binary: 2.4431 | Regression_cos_phi: 0.4953 | Regression_energy: 1.4390 | Regression_eta: 0.4631 | Regression_pt: 1.5145 | Regression_sin_phi: 0.3267 | LR: 3.48e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5600s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0062 | max_backward: 0.2031 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.3116 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5309.7 | uss_MB: 4549.2 | child_rss_MB: 3994.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:13:26,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:07,371] INFO : Step 32500/50000 rank0 | Train Loss: 7.7165 | Train Loss Unweighted: 2.7521 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.4544 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1298 | Regression_eta: 0.0121 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7453 | Classification_binary: 2.4544 | Regression_cos_phi: 0.5192 | Regression_energy: 1.4227 | Regression_eta: 0.5935 | Regression_pt: 1.4957 | Regression_sin_phi: 0.4857 | LR: 3.46e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7318s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3102 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3819 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5309.7 | uss_MB: 4549.3 | child_rss_MB: 3997.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:14:07,585] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:48,136] INFO : Step 32600/50000 rank0 | Train Loss: 7.7438 | Train Loss Unweighted: 2.7485 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4537 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1282 | Regression_eta: 0.0106 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8786 | Classification_binary: 2.4537 | Regression_cos_phi: 0.5952 | Regression_energy: 1.4043 | Regression_eta: 0.5222 | Regression_pt: 1.4603 | Regression_sin_phi: 0.4295 | LR: 3.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5582s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0127 | max_backward: 0.2026 | max_optimizer_step: 0.1373 | max_tensorboard_logging: 0.3030 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5309.7 | uss_MB: 4549.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:14:48,350] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:15:29,183] INFO : Step 32700/50000 rank0 | Train Loss: 7.6201 | Train Loss Unweighted: 2.6780 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3871 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1250 | Regression_eta: 0.0127 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8755 | Classification_binary: 2.3871 | Regression_cos_phi: 0.4957 | Regression_energy: 1.3691 | Regression_eta: 0.6224 | Regression_pt: 1.4475 | Regression_sin_phi: 0.4227 | LR: 3.41e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6466s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.3083 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.2894 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5309.7 | uss_MB: 4549.4 | child_rss_MB: 4023.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:15:29,396] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:11,415] INFO : Step 32800/50000 rank0 | Train Loss: 7.4295 | Train Loss Unweighted: 2.7360 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4418 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1285 | Regression_eta: 0.0098 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7995 | Classification_binary: 2.4418 | Regression_cos_phi: 0.4247 | Regression_energy: 1.4085 | Regression_eta: 0.4793 | Regression_pt: 1.5225 | Regression_sin_phi: 0.3531 | LR: 3.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6834s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1642 | max_loss: 0.0036 | max_backward: 0.2026 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5309.7 | uss_MB: 4549.5 | child_rss_MB: 3994.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:16:11,628] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:52,072] INFO : Step 32900/50000 rank0 | Train Loss: 8.4421 | Train Loss Unweighted: 2.8762 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5318 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1501 | Regression_eta: 0.0124 | Regression_pt: 0.1477 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9126 | Classification_binary: 2.5318 | Regression_cos_phi: 0.4950 | Regression_energy: 1.6441 | Regression_eta: 0.6101 | Regression_pt: 1.7717 | Regression_sin_phi: 0.4768 | LR: 3.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6271s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0014 | max_forward: 0.0958 | max_loss: 0.0036 | max_backward: 0.2026 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5310.7 | uss_MB: 4549.5 | child_rss_MB: 3994.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:16:52,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:17:33,811] INFO : Step 33000/50000 rank0 | Train Loss: 7.2547 | Train Loss Unweighted: 2.6361 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3567 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1210 | Regression_eta: 0.0128 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7709 | Classification_binary: 2.3567 | Regression_cos_phi: 0.3896 | Regression_energy: 1.3262 | Regression_eta: 0.6296 | Regression_pt: 1.4091 | Regression_sin_phi: 0.3726 | LR: 3.33e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8937s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0474 | max_forward: 0.1045 | max_loss: 0.0036 | max_backward: 0.3925 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.3935 | max_tensorboard_flush: 0.0133 | Memory: rss_MB: 5311.3 | uss_MB: 4551.4 | child_rss_MB: 3994.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:17:34,024] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:17:34,142] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 15:17:34,315] INFO : Running validation on rank0 +rank0 [2026-10-28 15:17:34,315] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:17:34,315] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:17:35,676] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 15:17:35,823] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:17:35,955] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:17:36,029] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:17:36,098] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:17:36,167] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:17:36,237] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:17:36,306] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:17:36,418] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:17:36,549] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:17:36,681] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:17:36,753] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:17:36,823] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:17:36,891] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:17:36,960] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:17:37,031] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:17:37,129] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:17:37,135] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:17:37,138] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:17:37,141] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:17:37,142] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:17:37,145] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:17:37,148] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:17:37,149] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:17:37,151] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:17:37,155] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:17:37,157] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:17:37,158] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:17:37,161] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:17:37,163] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:17:37,164] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:17:37,167] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:17:44,621] INFO : VALIDATION | Step=33000/50000 | Train Loss=7.2547 | Valid Loss=7.6175 | Train Loss Unweighted=2.6361 | Valid Loss Unweighted=2.7176 | Stale=0 | ETA=126.3m +rank0 [2026-10-28 15:17:44,630] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:17:44,699] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:17:44,702] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:17:44,871] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:17:46,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:17:47,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:17:48,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:17:48,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:17:49,011] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:17:49,014] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:17:49,083] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:17:49,086] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:17:49,188] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:17:51,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:17:51,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:17:52,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:17:53,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:17:53,248] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:17:53,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:17:53,315] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:17:53,318] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:17:53,420] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:17:55,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:17:55,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:17:56,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:17:57,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:17:57,303] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:17:57,310] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:17:57,569] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:17:57,974] INFO : Plotted number of elements +rank0 [2026-10-28 15:17:59,416] INFO : Plotted jets +rank0 [2026-10-28 15:18:00,512] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:00,544] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:00,772] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:01,161] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:03,093] INFO : Plotted jets +rank0 [2026-10-28 15:18:04,210] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:04,242] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 15:18:04,447] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 15:18:04,807] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:06,184] INFO : Plotted jets +rank0 [2026-10-28 15:18:07,251] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:48,270] INFO : Step 33100/50000 rank0 | Train Loss: 7.5267 | Train Loss Unweighted: 2.7300 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.4449 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1232 | Regression_eta: 0.0131 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7898 | Classification_binary: 2.4449 | Regression_cos_phi: 0.4579 | Regression_energy: 1.3503 | Regression_eta: 0.6430 | Regression_pt: 1.4223 | Regression_sin_phi: 0.4185 | LR: 3.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7263s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.2668 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3931 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5508.5 | uss_MB: 4746.5 | child_rss_MB: 3994.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:18:48,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:19:29,592] INFO : Step 33200/50000 rank0 | Train Loss: 7.1180 | Train Loss Unweighted: 2.6548 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.3807 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1188 | Regression_eta: 0.0092 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7422 | Classification_binary: 2.3807 | Regression_cos_phi: 0.4525 | Regression_energy: 1.3019 | Regression_eta: 0.4526 | Regression_pt: 1.4108 | Regression_sin_phi: 0.3774 | LR: 3.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7588s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3077 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.3882 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5508.5 | uss_MB: 4746.5 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:19:29,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:10,343] INFO : Step 33300/50000 rank0 | Train Loss: 7.1069 | Train Loss Unweighted: 2.6617 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3869 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1204 | Regression_eta: 0.0080 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8060 | Classification_binary: 2.3869 | Regression_cos_phi: 0.4382 | Regression_energy: 1.3194 | Regression_eta: 0.3936 | Regression_pt: 1.4051 | Regression_sin_phi: 0.3577 | LR: 3.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7273s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.2806 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.3853 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5508.5 | uss_MB: 4746.5 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:20:10,556] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:50,957] INFO : Step 33400/50000 rank0 | Train Loss: 8.0608 | Train Loss Unweighted: 2.7959 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4736 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1406 | Regression_eta: 0.0097 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9145 | Classification_binary: 2.4736 | Regression_cos_phi: 0.4850 | Regression_energy: 1.5411 | Regression_eta: 0.4746 | Regression_pt: 1.6421 | Regression_sin_phi: 0.5299 | LR: 3.23e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7485s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3078 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3901 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5388.5 | uss_MB: 4625.5 | child_rss_MB: 3994.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:20:51,168] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:21:31,963] INFO : Step 33500/50000 rank0 | Train Loss: 7.7721 | Train Loss Unweighted: 2.6624 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3629 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1282 | Regression_eta: 0.0153 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8123 | Classification_binary: 2.3629 | Regression_cos_phi: 0.4807 | Regression_energy: 1.4052 | Regression_eta: 0.7503 | Regression_pt: 1.4892 | Regression_sin_phi: 0.4716 | LR: 3.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7035s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1613 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0130 | Memory: rss_MB: 5388.5 | uss_MB: 4625.6 | child_rss_MB: 3991.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:21:32,176] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:12,658] INFO : Step 33600/50000 rank0 | Train Loss: 8.1198 | Train Loss Unweighted: 2.7941 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4647 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1458 | Regression_eta: 0.0129 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8506 | Classification_binary: 2.4647 | Regression_cos_phi: 0.4983 | Regression_energy: 1.5981 | Regression_eta: 0.6358 | Regression_pt: 1.6630 | Regression_sin_phi: 0.4094 | LR: 3.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7697s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1853 | max_loss: 0.0033 | max_backward: 0.2021 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.2987 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5388.5 | uss_MB: 4625.6 | child_rss_MB: 3995.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:22:12,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:54,418] INFO : Step 33700/50000 rank0 | Train Loss: 7.4886 | Train Loss Unweighted: 2.8212 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.5285 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1272 | Regression_eta: 0.0099 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7640 | Classification_binary: 2.5285 | Regression_cos_phi: 0.4268 | Regression_energy: 1.3937 | Regression_eta: 0.4844 | Regression_pt: 1.5277 | Regression_sin_phi: 0.3635 | LR: 3.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7703s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0206 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3073 | max_optimizer_step: 0.1420 | max_tensorboard_logging: 0.2974 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5388.5 | uss_MB: 4625.3 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:22:54,633] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:23:35,318] INFO : Step 33800/50000 rank0 | Train Loss: 7.7957 | Train Loss Unweighted: 2.7921 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4685 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1439 | Regression_eta: 0.0090 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8482 | Classification_binary: 2.4685 | Regression_cos_phi: 0.4039 | Regression_energy: 1.5769 | Regression_eta: 0.4435 | Regression_pt: 1.6884 | Regression_sin_phi: 0.3663 | LR: 3.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6926s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0719 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3070 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5388.5 | uss_MB: 4625.4 | child_rss_MB: 4021.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:23:35,529] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:16,273] INFO : Step 33900/50000 rank0 | Train Loss: 7.8186 | Train Loss Unweighted: 2.6774 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3708 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1318 | Regression_eta: 0.0162 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8111 | Classification_binary: 2.3708 | Regression_cos_phi: 0.4453 | Regression_energy: 1.4446 | Regression_eta: 0.7982 | Regression_pt: 1.5381 | Regression_sin_phi: 0.4104 | LR: 3.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7369s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.3037 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3913 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5388.5 | uss_MB: 4625.3 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:24:16,486] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:56,621] INFO : Step 34000/50000 rank0 | Train Loss: 8.1339 | Train Loss Unweighted: 2.7972 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4808 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1376 | Regression_eta: 0.0141 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8995 | Classification_binary: 2.4808 | Regression_cos_phi: 0.5625 | Regression_energy: 1.5077 | Regression_eta: 0.6955 | Regression_pt: 1.5660 | Regression_sin_phi: 0.4218 | LR: 3.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6755s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1043 | max_loss: 0.0035 | max_backward: 0.2025 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.3045 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5387.9 | uss_MB: 4625.0 | child_rss_MB: 3990.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:24:56,835] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:56,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 15:24:57,031] INFO : Running validation on rank0 +rank0 [2026-10-28 15:24:57,031] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:24:57,031] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:24:58,366] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 15:24:58,512] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:24:58,646] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:24:58,720] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:24:58,790] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:24:58,859] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:24:58,928] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:24:58,999] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:24:59,112] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:24:59,245] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:24:59,379] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:24:59,451] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:24:59,521] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:24:59,590] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:24:59,660] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:24:59,730] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:24:59,827] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:24:59,833] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:24:59,836] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:24:59,839] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:24:59,840] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:24:59,843] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:24:59,844] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:24:59,847] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:24:59,848] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:24:59,850] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:24:59,853] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:24:59,854] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:24:59,856] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:24:59,857] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:24:59,860] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:24:59,863] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:25:07,587] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.1339 | Valid Loss=7.5847 | Train Loss Unweighted=2.7972 | Valid Loss Unweighted=2.7077 | Stale=0 | ETA=118.9m +rank0 [2026-10-28 15:25:07,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:07,656] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:25:07,659] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:25:07,762] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:09,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:10,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:11,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:11,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:12,146] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:12,149] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:12,221] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:25:12,224] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:12,317] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:14,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:14,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:15,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:16,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:16,247] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:25:16,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:16,312] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:25:16,315] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:25:16,421] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:18,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:19,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:19,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:20,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:20,443] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:20,451] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:25:20,711] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:25:21,127] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:22,563] INFO : Plotted jets +rank0 [2026-10-28 15:25:24,259] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:24,291] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:24,589] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:24,984] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:26,378] INFO : Plotted jets +rank0 [2026-10-28 15:25:27,501] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:27,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 15:25:27,745] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 15:25:28,108] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:29,494] INFO : Plotted jets +rank0 [2026-10-28 15:25:30,555] INFO : Plotted jet ratio +rank0 [2026-10-28 15:26:12,377] INFO : Step 34100/50000 rank0 | Train Loss: 7.5329 | Train Loss Unweighted: 2.7521 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4639 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1254 | Regression_eta: 0.0110 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8241 | Classification_binary: 2.4639 | Regression_cos_phi: 0.4626 | Regression_energy: 1.3741 | Regression_eta: 0.5430 | Regression_pt: 1.4480 | Regression_sin_phi: 0.4171 | LR: 3.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7530s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1854 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2375 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5599.2 | uss_MB: 4836.1 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:26:12,592] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:26:53,982] INFO : Step 34200/50000 rank0 | Train Loss: 7.8139 | Train Loss Unweighted: 2.7777 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4532 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1433 | Regression_eta: 0.0082 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7757 | Classification_binary: 2.4532 | Regression_cos_phi: 0.4468 | Regression_energy: 1.5704 | Regression_eta: 0.4017 | Regression_pt: 1.7107 | Regression_sin_phi: 0.4553 | LR: 3.04e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8173s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0611 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3086 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2895 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5599.2 | uss_MB: 4835.9 | child_rss_MB: 3991.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:26:54,199] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:27:35,069] INFO : Step 34300/50000 rank0 | Train Loss: 7.3762 | Train Loss Unweighted: 2.7042 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4327 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1165 | Regression_eta: 0.0119 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7962 | Classification_binary: 2.4327 | Regression_cos_phi: 0.4959 | Regression_energy: 1.2760 | Regression_eta: 0.5842 | Regression_pt: 1.3416 | Regression_sin_phi: 0.4497 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7711s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2120 | max_loss: 0.0033 | max_backward: 0.2863 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 0.2857 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5599.2 | uss_MB: 4838.2 | child_rss_MB: 3994.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:27:35,282] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:28:16,832] INFO : Step 34400/50000 rank0 | Train Loss: 8.0568 | Train Loss Unweighted: 2.7796 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4552 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1418 | Regression_eta: 0.0140 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7782 | Classification_binary: 2.4552 | Regression_cos_phi: 0.5007 | Regression_energy: 1.5540 | Regression_eta: 0.6901 | Regression_pt: 1.6531 | Regression_sin_phi: 0.4255 | LR: 2.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7140s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2708 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5461.3 | uss_MB: 4698.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:28:17,045] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:28:57,673] INFO : Step 34500/50000 rank0 | Train Loss: 6.9764 | Train Loss Unweighted: 2.6937 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4157 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1235 | Regression_eta: 0.0068 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7795 | Classification_binary: 2.4157 | Regression_cos_phi: 0.3641 | Regression_energy: 1.3533 | Regression_eta: 0.3327 | Regression_pt: 1.4549 | Regression_sin_phi: 0.2761 | LR: 2.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6525s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1824 | max_loss: 0.0032 | max_backward: 0.2026 | max_optimizer_step: 0.1378 | max_tensorboard_logging: 0.2968 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5461.3 | uss_MB: 4698.5 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:28:57,898] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:29:38,428] INFO : Step 34600/50000 rank0 | Train Loss: 7.9621 | Train Loss Unweighted: 2.8351 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5358 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1284 | Regression_eta: 0.0127 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9445 | Classification_binary: 2.5358 | Regression_cos_phi: 0.4668 | Regression_energy: 1.4071 | Regression_eta: 0.6228 | Regression_pt: 1.4766 | Regression_sin_phi: 0.5085 | LR: 2.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7101s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0861 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3083 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5461.3 | uss_MB: 4698.5 | child_rss_MB: 4021.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:29:38,641] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:30:18,763] INFO : Step 34700/50000 rank0 | Train Loss: 7.4332 | Train Loss Unweighted: 2.7127 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.4199 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1278 | Regression_eta: 0.0069 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7871 | Classification_binary: 2.4199 | Regression_cos_phi: 0.5628 | Regression_energy: 1.4008 | Regression_eta: 0.3386 | Regression_pt: 1.5191 | Regression_sin_phi: 0.4049 | LR: 2.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5937s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0310 | max_backward: 0.2025 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3098 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5461.3 | uss_MB: 4698.6 | child_rss_MB: 3993.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:30:18,977] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:00,858] INFO : Step 34800/50000 rank0 | Train Loss: 7.4785 | Train Loss Unweighted: 2.5928 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2955 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1285 | Regression_eta: 0.0088 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8684 | Classification_binary: 2.2955 | Regression_cos_phi: 0.5802 | Regression_energy: 1.4076 | Regression_eta: 0.4349 | Regression_pt: 1.5253 | Regression_sin_phi: 0.3666 | LR: 2.90e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.8628s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0200 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.3900 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5462.3 | uss_MB: 4701.1 | child_rss_MB: 3995.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 15:31:01,073] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:41,917] INFO : Step 34900/50000 rank0 | Train Loss: 7.8591 | Train Loss Unweighted: 2.7561 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4324 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1426 | Regression_eta: 0.0112 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7775 | Classification_binary: 2.4324 | Regression_cos_phi: 0.4808 | Regression_energy: 1.5620 | Regression_eta: 0.5490 | Regression_pt: 1.6841 | Regression_sin_phi: 0.3732 | LR: 2.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7283s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0032 | max_backward: 0.2023 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5461.3 | uss_MB: 4698.9 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:31:42,145] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:23,135] INFO : Step 35000/50000 rank0 | Train Loss: 7.7028 | Train Loss Unweighted: 2.7187 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4207 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1300 | Regression_eta: 0.0115 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7966 | Classification_binary: 2.4207 | Regression_cos_phi: 0.5441 | Regression_energy: 1.4248 | Regression_eta: 0.5654 | Regression_pt: 1.4905 | Regression_sin_phi: 0.4607 | LR: 2.85e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8462s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1042 | max_loss: 0.0034 | max_backward: 0.3926 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5460.7 | uss_MB: 4700.6 | child_rss_MB: 3999.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:32:23,348] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:23,369] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 15:32:23,547] INFO : Running validation on rank0 +rank0 [2026-10-28 15:32:23,548] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:32:23,548] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:32:24,964] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 15:32:25,111] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:32:25,244] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:32:25,319] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:32:25,390] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:32:25,459] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:32:25,529] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:32:25,598] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:32:25,712] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:32:25,843] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:32:25,975] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:32:26,047] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:32:26,117] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:32:26,186] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:32:26,256] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:32:26,327] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:32:26,427] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:32:26,433] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:32:26,436] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:32:26,439] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:32:26,440] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:32:26,442] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:32:26,444] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:32:26,446] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:32:26,449] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:32:26,452] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:32:26,453] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:32:26,455] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:32:26,458] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:32:26,459] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:32:26,461] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:32:26,463] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:32:34,169] INFO : VALIDATION | Step=35000/50000 | Train Loss=7.7028 | Valid Loss=7.5819 | Train Loss Unweighted=2.7187 | Valid Loss Unweighted=2.6987 | Stale=0 | ETA=111.4m +rank0 [2026-10-28 15:32:34,179] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:34,254] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:32:34,258] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:32:34,431] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:36,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:37,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:38,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:38,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:38,867] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:38,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:38,966] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:32:38,976] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:39,159] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:41,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:41,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:42,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:43,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:43,538] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:43,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:43,620] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:32:43,623] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:32:43,804] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:45,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:46,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:47,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:47,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:47,809] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:47,817] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:32:48,074] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:32:48,481] INFO : Plotted number of elements +rank0 [2026-10-28 15:32:49,930] INFO : Plotted jets +rank0 [2026-10-28 15:32:51,032] INFO : Plotted jet ratio +rank0 [2026-10-28 15:32:51,066] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:51,292] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:51,685] INFO : Plotted number of elements +rank0 [2026-10-28 15:32:53,625] INFO : Plotted jets +rank0 [2026-10-28 15:32:54,745] INFO : Plotted jet ratio +rank0 [2026-10-28 15:32:54,779] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 15:32:54,989] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 15:32:55,353] INFO : Plotted number of elements +rank0 [2026-10-28 15:32:56,732] INFO : Plotted jets +rank0 [2026-10-28 15:32:57,803] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:39,450] INFO : Step 35100/50000 rank0 | Train Loss: 7.6070 | Train Loss Unweighted: 2.6246 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3332 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1242 | Regression_eta: 0.0143 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8692 | Classification_binary: 2.3332 | Regression_cos_phi: 0.5004 | Regression_energy: 1.3608 | Regression_eta: 0.7024 | Regression_pt: 1.4499 | Regression_sin_phi: 0.3911 | LR: 2.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.0018s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0936 | max_forward: 0.1044 | max_loss: 0.0035 | max_backward: 0.3943 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.3961 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5641.5 | uss_MB: 4884.5 | child_rss_MB: 4002.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:33:39,664] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 2% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:34:19,661] INFO : Step 35200/50000 rank0 | Train Loss: 6.8227 | Train Loss Unweighted: 2.5979 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.3362 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1161 | Regression_eta: 0.0095 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7395 | Classification_binary: 2.3362 | Regression_cos_phi: 0.3644 | Regression_energy: 1.2720 | Regression_eta: 0.4676 | Regression_pt: 1.3170 | Regression_sin_phi: 0.3259 | LR: 2.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6228s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0033 | max_backward: 0.2025 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5641.5 | uss_MB: 4882.4 | child_rss_MB: 4000.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:34:19,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:01,017] INFO : Step 35300/50000 rank0 | Train Loss: 7.1455 | Train Loss Unweighted: 2.6649 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3932 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1179 | Regression_eta: 0.0090 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7866 | Classification_binary: 2.3932 | Regression_cos_phi: 0.4653 | Regression_energy: 1.2917 | Regression_eta: 0.4430 | Regression_pt: 1.3815 | Regression_sin_phi: 0.3842 | LR: 2.79e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7563s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1826 | max_loss: 0.0037 | max_backward: 0.2025 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.2976 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5641.5 | uss_MB: 4882.4 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:35:01,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:42,787] INFO : Step 35400/50000 rank0 | Train Loss: 7.1655 | Train Loss Unweighted: 2.6257 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3535 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1184 | Regression_eta: 0.0084 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8892 | Classification_binary: 2.3535 | Regression_cos_phi: 0.4371 | Regression_energy: 1.2971 | Regression_eta: 0.4145 | Regression_pt: 1.3609 | Regression_sin_phi: 0.4132 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7660s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3042 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.4059 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5529.2 | uss_MB: 4769.4 | child_rss_MB: 3996.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:35:43,001] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:36:23,727] INFO : Step 35500/50000 rank0 | Train Loss: 7.6174 | Train Loss Unweighted: 2.7660 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4689 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1311 | Regression_eta: 0.0098 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8402 | Classification_binary: 2.4689 | Regression_cos_phi: 0.4731 | Regression_energy: 1.4369 | Regression_eta: 0.4807 | Regression_pt: 1.4946 | Regression_sin_phi: 0.4231 | LR: 2.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7564s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2101 | max_loss: 0.0034 | max_backward: 0.2868 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5529.2 | uss_MB: 4772.9 | child_rss_MB: 3995.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 15:36:23,940] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:05,210] INFO : Step 35600/50000 rank0 | Train Loss: 7.5924 | Train Loss Unweighted: 2.6903 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3821 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1348 | Regression_eta: 0.0109 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7928 | Classification_binary: 2.3821 | Regression_cos_phi: 0.3767 | Regression_energy: 1.4765 | Regression_eta: 0.5382 | Regression_pt: 1.5986 | Regression_sin_phi: 0.4276 | LR: 2.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7339s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0879 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.3104 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5529.2 | uss_MB: 4771.0 | child_rss_MB: 4028.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:37:05,426] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:46,127] INFO : Step 35700/50000 rank0 | Train Loss: 7.4640 | Train Loss Unweighted: 2.7411 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4584 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1229 | Regression_eta: 0.0073 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7987 | Classification_binary: 2.4584 | Regression_cos_phi: 0.5640 | Regression_energy: 1.3463 | Regression_eta: 0.3594 | Regression_pt: 1.4306 | Regression_sin_phi: 0.5066 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7552s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0034 | max_backward: 0.3108 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3810 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5529.2 | uss_MB: 4771.3 | child_rss_MB: 4001.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:37:46,339] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:38:27,035] INFO : Step 35800/50000 rank0 | Train Loss: 7.4530 | Train Loss Unweighted: 2.6901 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3893 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1338 | Regression_eta: 0.0079 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8189 | Classification_binary: 2.3893 | Regression_cos_phi: 0.4781 | Regression_energy: 1.4661 | Regression_eta: 0.3897 | Regression_pt: 1.5444 | Regression_sin_phi: 0.3665 | LR: 2.68e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7834s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1044 | max_loss: 0.0034 | max_backward: 0.3855 | max_optimizer_step: 0.1399 | max_tensorboard_logging: 0.3330 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5529.2 | uss_MB: 4773.4 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:38:27,246] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:07,221] INFO : Step 35900/50000 rank0 | Train Loss: 7.1536 | Train Loss Unweighted: 2.6729 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3892 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1245 | Regression_eta: 0.0069 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8754 | Classification_binary: 2.3892 | Regression_cos_phi: 0.3500 | Regression_energy: 1.3645 | Regression_eta: 0.3386 | Regression_pt: 1.4721 | Regression_sin_phi: 0.3640 | LR: 2.65e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6332s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0907 | max_loss: 0.0033 | max_backward: 0.2022 | max_optimizer_step: 0.2233 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5529.2 | uss_MB: 4771.3 | child_rss_MB: 3998.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:39:07,437] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:47,648] INFO : Step 36000/50000 rank0 | Train Loss: 8.2734 | Train Loss Unweighted: 2.8643 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5492 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1359 | Regression_eta: 0.0184 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8597 | Classification_binary: 2.5492 | Regression_cos_phi: 0.5455 | Regression_energy: 1.4886 | Regression_eta: 0.9042 | Regression_pt: 1.5417 | Regression_sin_phi: 0.3845 | LR: 2.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6587s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2106 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.3959 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5528.6 | uss_MB: 4769.4 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:39:47,863] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:47,908] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 15:39:48,089] INFO : Running validation on rank0 +rank0 [2026-10-28 15:39:48,090] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:39:48,090] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:39:49,417] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 15:39:49,564] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:39:49,696] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:39:49,770] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:39:49,839] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:39:49,908] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:39:49,977] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:39:50,046] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:39:50,160] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:39:50,294] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:39:50,426] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:39:50,497] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:39:50,566] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:39:50,634] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:39:50,703] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:39:50,772] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:39:50,872] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:39:50,878] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:39:50,881] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:39:50,882] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:39:50,885] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:39:50,887] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:39:50,889] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:39:50,891] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:39:50,894] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:39:50,895] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:39:50,897] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:39:50,900] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:39:50,901] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:39:50,904] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:39:50,906] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:39:50,909] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:39:58,521] INFO : VALIDATION | Step=36000/50000 | Train Loss=8.2734 | Valid Loss=7.5099 | Train Loss Unweighted=2.8643 | Valid Loss Unweighted=2.6828 | Stale=0 | ETA=104.0m +rank0 [2026-10-28 15:39:58,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:39:58,586] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:39:58,590] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:39:58,764] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:00,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:01,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:01,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:02,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:02,806] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:02,809] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:02,875] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:40:02,878] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:02,975] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:04,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:05,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:06,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:06,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:07,010] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:07,013] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:07,084] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:40:07,087] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:40:07,186] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:09,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:09,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:10,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:11,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:11,164] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:11,173] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:40:11,426] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:40:11,829] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:13,263] INFO : Plotted jets +rank0 [2026-10-28 15:40:14,358] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:14,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:14,616] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:15,003] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:16,942] INFO : Plotted jets +rank0 [2026-10-28 15:40:18,059] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:18,092] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 15:40:18,298] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 15:40:18,661] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:20,039] INFO : Plotted jets +rank0 [2026-10-28 15:40:21,112] INFO : Plotted jet ratio +rank0 [2026-10-28 15:41:03,045] INFO : Step 36100/50000 rank0 | Train Loss: 7.1912 | Train Loss Unweighted: 2.7191 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4420 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1222 | Regression_eta: 0.0092 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8289 | Classification_binary: 2.4420 | Regression_cos_phi: 0.3668 | Regression_energy: 1.3386 | Regression_eta: 0.4547 | Regression_pt: 1.4014 | Regression_sin_phi: 0.3588 | LR: 2.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7572s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5736.2 | uss_MB: 4976.1 | child_rss_MB: 3998.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:41:03,260] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:41:43,455] INFO : Step 36200/50000 rank0 | Train Loss: 6.9320 | Train Loss Unweighted: 2.5627 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.2922 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1171 | Regression_eta: 0.0069 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8457 | Classification_binary: 2.2922 | Regression_cos_phi: 0.3791 | Regression_energy: 1.2836 | Regression_eta: 0.3396 | Regression_pt: 1.3984 | Regression_sin_phi: 0.3935 | LR: 2.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6833s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.2286 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5736.2 | uss_MB: 4976.1 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:41:43,666] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:42:24,804] INFO : Step 36300/50000 rank0 | Train Loss: 7.5767 | Train Loss Unweighted: 2.6947 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3701 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1466 | Regression_eta: 0.0070 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7749 | Classification_binary: 2.3701 | Regression_cos_phi: 0.3863 | Regression_energy: 1.6064 | Regression_eta: 0.3421 | Regression_pt: 1.7117 | Regression_sin_phi: 0.3852 | LR: 2.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3090 | max_optimizer_step: 0.1432 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5736.2 | uss_MB: 4976.1 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:42:25,018] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:06,128] INFO : Step 36400/50000 rank0 | Train Loss: 7.1193 | Train Loss Unweighted: 2.6120 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3425 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1160 | Regression_eta: 0.0089 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8668 | Classification_binary: 2.3425 | Regression_cos_phi: 0.4585 | Regression_energy: 1.2706 | Regression_eta: 0.4387 | Regression_pt: 1.3590 | Regression_sin_phi: 0.3831 | LR: 2.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8124s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0109 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.3105 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.4086 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5604.2 | uss_MB: 4841.4 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:43:06,341] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:46,922] INFO : Step 36500/50000 rank0 | Train Loss: 7.6218 | Train Loss Unweighted: 2.6561 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3535 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1325 | Regression_eta: 0.0108 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8798 | Classification_binary: 2.3535 | Regression_cos_phi: 0.4333 | Regression_energy: 1.4515 | Regression_eta: 0.5305 | Regression_pt: 1.5242 | Regression_sin_phi: 0.4491 | LR: 2.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9065s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0307 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3939 | max_optimizer_step: 0.1436 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5604.2 | uss_MB: 4844.8 | child_rss_MB: 3996.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:43:47,139] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:44:27,277] INFO : Step 36600/50000 rank0 | Train Loss: 7.9635 | Train Loss Unweighted: 2.7214 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4120 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1345 | Regression_eta: 0.0129 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8571 | Classification_binary: 2.4120 | Regression_cos_phi: 0.5791 | Regression_energy: 1.4733 | Regression_eta: 0.6365 | Regression_pt: 1.5330 | Regression_sin_phi: 0.4725 | LR: 2.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6316s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0033 | max_backward: 0.2022 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5604.2 | uss_MB: 4843.3 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:44:27,491] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:08,462] INFO : Step 36700/50000 rank0 | Train Loss: 8.4009 | Train Loss Unweighted: 2.8552 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5246 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1431 | Regression_eta: 0.0174 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8535 | Classification_binary: 2.5246 | Regression_cos_phi: 0.4921 | Regression_energy: 1.5678 | Regression_eta: 0.8559 | Regression_pt: 1.6487 | Regression_sin_phi: 0.4583 | LR: 2.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.1019 | max_backward: 0.2026 | max_optimizer_step: 0.1410 | max_tensorboard_logging: 0.2998 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5605.2 | uss_MB: 4843.5 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:45:08,675] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:50,442] INFO : Step 36800/50000 rank0 | Train Loss: 7.6975 | Train Loss Unweighted: 2.8132 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5192 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1290 | Regression_eta: 0.0089 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8190 | Classification_binary: 2.5192 | Regression_cos_phi: 0.5984 | Regression_energy: 1.4138 | Regression_eta: 0.4357 | Regression_pt: 1.4757 | Regression_sin_phi: 0.4357 | LR: 2.46e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9258s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0417 | max_forward: 0.1038 | max_loss: 0.0033 | max_backward: 0.3930 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.3963 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 5605.2 | uss_MB: 4845.7 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 15:45:50,656] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:46:31,508] INFO : Step 36900/50000 rank0 | Train Loss: 7.5002 | Train Loss Unweighted: 2.7044 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4188 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1235 | Regression_eta: 0.0115 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8139 | Classification_binary: 2.4188 | Regression_cos_phi: 0.5367 | Regression_energy: 1.3529 | Regression_eta: 0.5636 | Regression_pt: 1.4319 | Regression_sin_phi: 0.3824 | LR: 2.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7662s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1854 | max_loss: 0.0035 | max_backward: 0.2024 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5605.2 | uss_MB: 4843.6 | child_rss_MB: 4000.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:46:31,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:12,436] INFO : Step 37000/50000 rank0 | Train Loss: 7.6896 | Train Loss Unweighted: 2.7126 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4072 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1345 | Regression_eta: 0.0114 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7726 | Classification_binary: 2.4072 | Regression_cos_phi: 0.4795 | Regression_energy: 1.4743 | Regression_eta: 0.5589 | Regression_pt: 1.5460 | Regression_sin_phi: 0.4511 | LR: 2.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.9567s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1987 | max_loss: 0.0033 | max_backward: 0.3922 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.2924 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 5604.0 | uss_MB: 4843.3 | child_rss_MB: 4000.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:47:12,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:12,668] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 15:47:12,947] INFO : Running validation on rank0 +rank0 [2026-10-28 15:47:12,947] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:47:12,948] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:47:14,390] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 15:47:14,536] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:47:14,668] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:47:14,742] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:47:14,811] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:47:14,880] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:47:14,949] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:47:15,018] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:47:15,131] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:47:15,263] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:47:15,394] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:47:15,466] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:47:15,535] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:47:15,604] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:47:15,672] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:47:15,742] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:47:15,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:47:15,850] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:47:15,853] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:47:15,856] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:47:15,857] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:47:15,859] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:47:15,861] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:47:15,863] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:47:15,864] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:47:15,867] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:47:15,870] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:47:15,871] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:47:15,873] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:47:15,876] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:47:15,877] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:47:15,880] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:47:23,634] INFO : VALIDATION | Step=37000/50000 | Train Loss=7.6896 | Valid Loss=7.5400 | Train Loss Unweighted=2.7126 | Valid Loss Unweighted=2.6929 | Stale=1 | ETA=96.6m +rank0 [2026-10-28 15:47:23,644] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:23,732] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:47:23,736] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:47:23,950] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:25,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:26,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:27,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:28,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:28,183] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:28,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:28,243] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:47:28,246] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:28,355] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:30,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:30,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:31,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:32,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:32,367] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:32,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:32,441] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:47:32,444] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:47:32,544] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:34,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:34,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:35,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:36,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:36,467] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:47:36,476] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:47:36,738] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:47:37,144] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:38,585] INFO : Plotted jets +rank0 [2026-10-28 15:47:39,680] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:39,712] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:39,942] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:40,334] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:42,283] INFO : Plotted jets +rank0 [2026-10-28 15:47:43,410] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:43,444] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 15:47:43,651] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 15:47:44,011] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:45,617] INFO : Plotted jets +rank0 [2026-10-28 15:47:46,691] INFO : Plotted jet ratio +rank0 [2026-10-28 15:48:27,765] INFO : Step 37100/50000 rank0 | Train Loss: 7.5056 | Train Loss Unweighted: 2.7824 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4934 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1251 | Regression_eta: 0.0073 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8272 | Classification_binary: 2.4934 | Regression_cos_phi: 0.5464 | Regression_energy: 1.3705 | Regression_eta: 0.3587 | Regression_pt: 1.4926 | Regression_sin_phi: 0.4169 | LR: 2.40e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6667s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0814 | max_backward: 0.2029 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5812.9 | uss_MB: 5050.8 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:48:27,978] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:09,103] INFO : Step 37200/50000 rank0 | Train Loss: 7.2977 | Train Loss Unweighted: 2.5856 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.3135 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1166 | Regression_eta: 0.0115 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9662 | Classification_binary: 2.3135 | Regression_cos_phi: 0.4549 | Regression_energy: 1.2774 | Regression_eta: 0.5659 | Regression_pt: 1.3239 | Regression_sin_phi: 0.3959 | LR: 2.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6228s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0654 | max_backward: 0.2029 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5812.9 | uss_MB: 5050.8 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:49:09,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:50,309] INFO : Step 37300/50000 rank0 | Train Loss: 7.0843 | Train Loss Unweighted: 2.6577 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3738 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1256 | Regression_eta: 0.0086 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7711 | Classification_binary: 2.3738 | Regression_cos_phi: 0.3363 | Regression_energy: 1.3766 | Regression_eta: 0.4212 | Regression_pt: 1.4769 | Regression_sin_phi: 0.3285 | LR: 2.36e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0228 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3115 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.4060 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 5812.9 | uss_MB: 5050.8 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:49:50,521] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:50:31,240] INFO : Step 37400/50000 rank0 | Train Loss: 7.8018 | Train Loss Unweighted: 2.5691 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.2703 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1265 | Regression_eta: 0.0260 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7672 | Classification_binary: 2.2703 | Regression_cos_phi: 0.3586 | Regression_energy: 1.3856 | Regression_eta: 1.2767 | Regression_pt: 1.4372 | Regression_sin_phi: 0.3061 | LR: 2.34e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0874 | max_backward: 0.2027 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3526 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5679.6 | uss_MB: 4916.3 | child_rss_MB: 4000.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:50:31,454] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:51:13,364] INFO : Step 37500/50000 rank0 | Train Loss: 8.2378 | Train Loss Unweighted: 2.8042 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4701 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1472 | Regression_eta: 0.0102 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8889 | Classification_binary: 2.4701 | Regression_cos_phi: 0.5466 | Regression_energy: 1.6133 | Regression_eta: 0.5029 | Regression_pt: 1.6965 | Regression_sin_phi: 0.5195 | LR: 2.32e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.7955s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0785 | max_loss: 0.0037 | max_backward: 0.3101 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4010 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5680.6 | uss_MB: 4917.4 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:51:13,577] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:51:55,509] INFO : Step 37600/50000 rank0 | Train Loss: 7.1560 | Train Loss Unweighted: 2.6325 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3552 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1196 | Regression_eta: 0.0073 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7977 | Classification_binary: 2.3552 | Regression_cos_phi: 0.5509 | Regression_energy: 1.3108 | Regression_eta: 0.3567 | Regression_pt: 1.4378 | Regression_sin_phi: 0.3469 | LR: 2.30e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7320s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2479 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.3941 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5680.6 | uss_MB: 4917.7 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:51:55,722] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:52:37,053] INFO : Step 37700/50000 rank0 | Train Loss: 7.2398 | Train Loss Unweighted: 2.5846 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.2939 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1279 | Regression_eta: 0.0085 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7781 | Classification_binary: 2.2939 | Regression_cos_phi: 0.4677 | Regression_energy: 1.4019 | Regression_eta: 0.4188 | Regression_pt: 1.4961 | Regression_sin_phi: 0.3833 | LR: 2.28e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6899s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1707 | max_loss: 0.0034 | max_backward: 0.2034 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.2805 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5680.6 | uss_MB: 4917.9 | child_rss_MB: 4000.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:52:37,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:18,803] INFO : Step 37800/50000 rank0 | Train Loss: 7.9413 | Train Loss Unweighted: 2.7216 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4165 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1356 | Regression_eta: 0.0162 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8250 | Classification_binary: 2.4165 | Regression_cos_phi: 0.5321 | Regression_energy: 1.4853 | Regression_eta: 0.7951 | Regression_pt: 1.4517 | Regression_sin_phi: 0.4355 | LR: 2.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7626s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3030 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.3967 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 5680.6 | uss_MB: 4918.0 | child_rss_MB: 4000.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:53:19,019] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:59,792] INFO : Step 37900/50000 rank0 | Train Loss: 7.6456 | Train Loss Unweighted: 2.6682 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3785 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1241 | Regression_eta: 0.0175 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7853 | Classification_binary: 2.3785 | Regression_cos_phi: 0.4732 | Regression_energy: 1.3597 | Regression_eta: 0.8623 | Regression_pt: 1.4236 | Regression_sin_phi: 0.3632 | LR: 2.24e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6336s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0438 | max_backward: 0.2027 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3151 | max_tensorboard_flush: 0.0166 | Memory: rss_MB: 5680.6 | uss_MB: 4918.0 | child_rss_MB: 3997.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:54:00,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:41,654] INFO : Step 38000/50000 rank0 | Train Loss: 7.0918 | Train Loss Unweighted: 2.7374 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.4624 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1201 | Regression_eta: 0.0067 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7422 | Classification_binary: 2.4624 | Regression_cos_phi: 0.4054 | Regression_energy: 1.3161 | Regression_eta: 0.3279 | Regression_pt: 1.4413 | Regression_sin_phi: 0.3966 | LR: 2.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8734s | Phase Times: max_pre_step_barrier: 0.1043 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3102 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.3933 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5678.9 | uss_MB: 4916.0 | child_rss_MB: 4000.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:54:41,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:41,964] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 15:54:42,144] INFO : Running validation on rank0 +rank0 [2026-10-28 15:54:42,144] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:54:42,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:54:43,454] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 15:54:43,600] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:54:43,733] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:54:43,808] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:54:43,876] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:54:43,945] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:54:44,014] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:54:44,084] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:54:44,198] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:54:44,330] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:54:44,462] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:54:44,533] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:54:44,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:54:44,669] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:54:44,739] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:54:44,809] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:54:44,909] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:54:44,915] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:54:44,918] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:54:44,921] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:54:44,922] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:54:44,924] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:54:44,926] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:54:44,928] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:54:44,929] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:54:44,932] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:54:44,935] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:54:44,935] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:54:44,938] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:54:44,941] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:54:44,942] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:54:44,945] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:54:52,403] INFO : VALIDATION | Step=38000/50000 | Train Loss=7.0918 | Valid Loss=7.4444 | Train Loss Unweighted=2.7374 | Valid Loss Unweighted=2.6633 | Stale=0 | ETA=89.2m +rank0 [2026-10-28 15:54:52,413] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:52,479] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:54:52,482] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:54:52,656] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:54,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:55,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:55,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:56,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:56,841] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:54:56,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:56,908] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:54:56,911] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:57,021] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:59,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:59,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:55:00,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:55:00,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:55:01,046] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:55:01,049] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:55:01,110] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:55:01,112] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:55:01,225] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:55:03,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:55:03,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:55:04,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:55:04,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:55:05,139] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:55:05,147] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:55:05,405] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:55:05,809] INFO : Plotted number of elements +rank0 [2026-10-28 15:55:07,254] INFO : Plotted jets +rank0 [2026-10-28 15:55:08,347] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:08,379] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:55:08,605] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:55:08,996] INFO : Plotted number of elements +rank0 [2026-10-28 15:55:10,946] INFO : Plotted jets +rank0 [2026-10-28 15:55:12,074] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:12,106] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 15:55:12,317] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 15:55:12,678] INFO : Plotted number of elements +rank0 [2026-10-28 15:55:14,058] INFO : Plotted jets +rank0 [2026-10-28 15:55:15,133] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:55,883] INFO : Step 38100/50000 rank0 | Train Loss: 7.4852 | Train Loss Unweighted: 2.6928 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3988 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1296 | Regression_eta: 0.0113 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8044 | Classification_binary: 2.3988 | Regression_cos_phi: 0.4349 | Regression_energy: 1.4204 | Regression_eta: 0.5554 | Regression_pt: 1.4791 | Regression_sin_phi: 0.3921 | LR: 2.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6589s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.3702 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5881.2 | uss_MB: 5120.1 | child_rss_MB: 4000.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:55:56,095] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:56:37,253] INFO : Step 38200/50000 rank0 | Train Loss: 7.6688 | Train Loss Unweighted: 2.7372 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.4314 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1346 | Regression_eta: 0.0093 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8020 | Classification_binary: 2.4314 | Regression_cos_phi: 0.4821 | Regression_energy: 1.4745 | Regression_eta: 0.4571 | Regression_pt: 1.5656 | Regression_sin_phi: 0.4561 | LR: 2.18e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6287s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0685 | max_backward: 0.2034 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.2924 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5881.2 | uss_MB: 5120.2 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:56:37,467] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:57:17,105] INFO : Step 38300/50000 rank0 | Train Loss: 7.5861 | Train Loss Unweighted: 2.6993 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4139 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1220 | Regression_eta: 0.0105 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7757 | Classification_binary: 2.4139 | Regression_cos_phi: 0.5521 | Regression_energy: 1.3366 | Regression_eta: 0.5154 | Regression_pt: 1.4332 | Regression_sin_phi: 0.5591 | LR: 2.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1062 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3111 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5881.2 | uss_MB: 5120.0 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:57:17,320] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:57:57,954] INFO : Step 38400/50000 rank0 | Train Loss: 7.0001 | Train Loss Unweighted: 2.6278 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3581 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1161 | Regression_eta: 0.0096 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7776 | Classification_binary: 2.3581 | Regression_cos_phi: 0.3480 | Regression_energy: 1.2717 | Regression_eta: 0.4702 | Regression_pt: 1.3965 | Regression_sin_phi: 0.3779 | LR: 2.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7657s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2778 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3982 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5751.1 | uss_MB: 4989.6 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:57:58,166] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:58:39,103] INFO : Step 38500/50000 rank0 | Train Loss: 7.5890 | Train Loss Unweighted: 2.7326 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4278 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1346 | Regression_eta: 0.0095 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 2.4278 | Regression_cos_phi: 0.4196 | Regression_energy: 1.4747 | Regression_eta: 0.4673 | Regression_pt: 1.5657 | Regression_sin_phi: 0.4203 | LR: 2.12e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7444s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.3110 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5751.1 | uss_MB: 4990.7 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:58:39,315] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:59:19,931] INFO : Step 38600/50000 rank0 | Train Loss: 7.0463 | Train Loss Unweighted: 2.6368 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3701 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1149 | Regression_eta: 0.0087 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8043 | Classification_binary: 2.3701 | Regression_cos_phi: 0.4542 | Regression_energy: 1.2591 | Regression_eta: 0.4255 | Regression_pt: 1.3599 | Regression_sin_phi: 0.3732 | LR: 2.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6931s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2179 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5751.1 | uss_MB: 4990.9 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:59:20,143] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:00,003] INFO : Step 38700/50000 rank0 | Train Loss: 7.2538 | Train Loss Unweighted: 2.6871 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.4079 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1193 | Regression_eta: 0.0119 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7689 | Classification_binary: 2.4079 | Regression_cos_phi: 0.3422 | Regression_energy: 1.3072 | Regression_eta: 0.5863 | Regression_pt: 1.4421 | Regression_sin_phi: 0.3991 | LR: 2.09e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5735s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2867 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5751.1 | uss_MB: 4991.0 | child_rss_MB: 3997.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:00:00,215] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:41,690] INFO : Step 38800/50000 rank0 | Train Loss: 7.3055 | Train Loss Unweighted: 2.6311 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3468 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1206 | Regression_eta: 0.0128 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8226 | Classification_binary: 2.3468 | Regression_cos_phi: 0.3684 | Regression_energy: 1.3211 | Regression_eta: 0.6313 | Regression_pt: 1.4681 | Regression_sin_phi: 0.3471 | LR: 2.07e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7120s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2550 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5751.1 | uss_MB: 4991.0 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:00:41,901] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:01:23,493] INFO : Step 38900/50000 rank0 | Train Loss: 7.2524 | Train Loss Unweighted: 2.6470 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3575 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1274 | Regression_eta: 0.0082 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8256 | Classification_binary: 2.3575 | Regression_cos_phi: 0.3620 | Regression_energy: 1.3957 | Regression_eta: 0.4053 | Regression_pt: 1.4918 | Regression_sin_phi: 0.4146 | LR: 2.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6649s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0046 | max_forward: 0.0770 | max_loss: 0.0034 | max_backward: 0.3102 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.2805 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5751.1 | uss_MB: 4991.1 | child_rss_MB: 4027.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:01:23,707] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:02:05,452] INFO : Step 39000/50000 rank0 | Train Loss: 6.8701 | Train Loss Unweighted: 2.5289 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.2705 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1099 | Regression_eta: 0.0097 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8401 | Classification_binary: 2.2705 | Regression_cos_phi: 0.4200 | Regression_energy: 1.2041 | Regression_eta: 0.4762 | Regression_pt: 1.3091 | Regression_sin_phi: 0.3501 | LR: 2.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7726s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.2103 | max_loss: 0.0036 | max_backward: 0.2866 | max_optimizer_step: 0.1420 | max_tensorboard_logging: 0.2865 | max_tensorboard_flush: 0.0081 | Memory: rss_MB: 5750.4 | uss_MB: 4991.7 | child_rss_MB: 3993.3 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:02:05,665] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:02:05,684] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 16:02:05,859] INFO : Running validation on rank0 +rank0 [2026-10-28 16:02:05,859] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:02:05,859] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:02:07,173] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 16:02:07,321] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:02:07,453] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:02:07,527] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:02:07,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:02:07,666] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:02:07,736] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:02:07,805] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:02:07,918] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:02:08,050] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:02:08,182] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:02:08,253] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:02:08,322] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:02:08,391] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:02:08,461] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:02:08,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:02:08,627] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:02:08,633] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:02:08,636] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:02:08,639] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:02:08,640] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:02:08,643] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:02:08,645] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:02:08,646] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:02:08,649] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:02:08,652] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:02:08,653] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:02:08,655] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:02:08,658] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:02:08,660] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:02:08,661] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:02:08,664] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:02:16,230] INFO : VALIDATION | Step=39000/50000 | Train Loss=6.8701 | Valid Loss=7.4523 | Train Loss Unweighted=2.5289 | Valid Loss Unweighted=2.6673 | Stale=1 | ETA=81.7m +rank0 [2026-10-28 16:02:16,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:16,296] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:02:16,299] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:02:16,474] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:02:18,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:02:19,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:02:19,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:02:20,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:02:20,815] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:02:20,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:20,874] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:02:20,877] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:20,996] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:02:23,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:02:23,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:02:24,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:02:24,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:02:25,038] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:02:25,041] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:25,096] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:02:25,099] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:02:25,224] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:02:26,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:02:27,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:02:28,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:02:28,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:02:29,146] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:02:29,154] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:02:29,419] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:02:29,841] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:31,286] INFO : Plotted jets +rank0 [2026-10-28 16:02:32,387] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:32,418] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:32,648] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:33,043] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:35,032] INFO : Plotted jets +rank0 [2026-10-28 16:02:36,157] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:36,190] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 16:02:36,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 16:02:36,763] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:38,144] INFO : Plotted jets +rank0 [2026-10-28 16:02:39,213] INFO : Plotted jet ratio +rank0 [2026-10-28 16:03:20,346] INFO : Step 39100/50000 rank0 | Train Loss: 7.4023 | Train Loss Unweighted: 2.6551 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3667 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1251 | Regression_eta: 0.0099 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8768 | Classification_binary: 2.3667 | Regression_cos_phi: 0.4153 | Regression_energy: 1.3706 | Regression_eta: 0.4859 | Regression_pt: 1.4620 | Regression_sin_phi: 0.4250 | LR: 2.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7490s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1048 | max_loss: 0.0036 | max_backward: 0.2868 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3106 | max_tensorboard_flush: 0.0148 | Memory: rss_MB: 5956.4 | uss_MB: 5197.1 | child_rss_MB: 3992.4 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:03:20,559] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:00,990] INFO : Step 39200/50000 rank0 | Train Loss: 7.6139 | Train Loss Unweighted: 2.7196 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4241 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1280 | Regression_eta: 0.0122 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8369 | Classification_binary: 2.4241 | Regression_cos_phi: 0.4937 | Regression_energy: 1.4024 | Regression_eta: 0.6010 | Regression_pt: 1.4921 | Regression_sin_phi: 0.3637 | LR: 2.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7094s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0014 | max_forward: 0.0771 | max_loss: 0.0035 | max_backward: 0.2431 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.3806 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5956.4 | uss_MB: 5195.1 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:04:01,202] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:42,599] INFO : Step 39300/50000 rank0 | Train Loss: 7.6549 | Train Loss Unweighted: 2.6411 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3272 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1395 | Regression_eta: 0.0089 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8008 | Classification_binary: 2.3272 | Regression_cos_phi: 0.5089 | Regression_energy: 1.5282 | Regression_eta: 0.4397 | Regression_pt: 1.6062 | Regression_sin_phi: 0.4439 | LR: 1.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6589s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1017 | max_loss: 0.0037 | max_backward: 0.2031 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5956.4 | uss_MB: 5195.1 | child_rss_MB: 3991.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:04:42,814] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:05:24,415] INFO : Step 39400/50000 rank0 | Train Loss: 6.9390 | Train Loss Unweighted: 2.6161 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.3452 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1191 | Regression_eta: 0.0096 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7551 | Classification_binary: 2.3452 | Regression_cos_phi: 0.3388 | Regression_energy: 1.3044 | Regression_eta: 0.4726 | Regression_pt: 1.3887 | Regression_sin_phi: 0.3341 | LR: 1.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9139s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3075 | max_optimizer_step: 0.1428 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5956.4 | uss_MB: 5195.1 | child_rss_MB: 3994.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:05:24,626] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:05,831] INFO : Step 39500/50000 rank0 | Train Loss: 7.2029 | Train Loss Unweighted: 2.6354 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.3512 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1278 | Regression_eta: 0.0091 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7484 | Classification_binary: 2.3512 | Regression_cos_phi: 0.4819 | Regression_energy: 1.4001 | Regression_eta: 0.4470 | Regression_pt: 1.4240 | Regression_sin_phi: 0.3502 | LR: 1.94e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.7741s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3080 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3830 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5826.7 | uss_MB: 5066.4 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:06:06,044] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:46,554] INFO : Step 39600/50000 rank0 | Train Loss: 7.4494 | Train Loss Unweighted: 2.6677 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3763 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1268 | Regression_eta: 0.0099 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7707 | Classification_binary: 2.3763 | Regression_cos_phi: 0.4801 | Regression_energy: 1.3893 | Regression_eta: 0.4865 | Regression_pt: 1.4877 | Regression_sin_phi: 0.4588 | LR: 1.93e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8199s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1041 | max_loss: 0.0034 | max_backward: 0.3932 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3418 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5826.7 | uss_MB: 5068.8 | child_rss_MB: 3998.4 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:06:46,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:07:28,572] INFO : Step 39700/50000 rank0 | Train Loss: 7.7855 | Train Loss Unweighted: 2.6558 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3532 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1324 | Regression_eta: 0.0163 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8073 | Classification_binary: 2.3532 | Regression_cos_phi: 0.5027 | Regression_energy: 1.4511 | Regression_eta: 0.8008 | Regression_pt: 1.4756 | Regression_sin_phi: 0.3948 | LR: 1.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3095 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.3683 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5826.7 | uss_MB: 5066.8 | child_rss_MB: 3996.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:07:28,782] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:10,058] INFO : Step 39800/50000 rank0 | Train Loss: 7.6931 | Train Loss Unweighted: 2.7466 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4372 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1365 | Regression_eta: 0.0115 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8739 | Classification_binary: 2.4372 | Regression_cos_phi: 0.3602 | Regression_energy: 1.4959 | Regression_eta: 0.5666 | Regression_pt: 1.5747 | Regression_sin_phi: 0.3846 | LR: 1.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7740s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1225 | max_loss: 0.0034 | max_backward: 0.3931 | max_optimizer_step: 0.1389 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5827.7 | uss_MB: 5069.1 | child_rss_MB: 4024.2 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:08:10,273] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:51,081] INFO : Step 39900/50000 rank0 | Train Loss: 7.4576 | Train Loss Unweighted: 2.5526 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.2656 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1233 | Regression_eta: 0.0139 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8015 | Classification_binary: 2.2656 | Regression_cos_phi: 0.4096 | Regression_energy: 1.3514 | Regression_eta: 0.6832 | Regression_pt: 1.4196 | Regression_sin_phi: 0.5267 | LR: 1.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5615s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0179 | max_backward: 0.2036 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5828.7 | uss_MB: 5067.0 | child_rss_MB: 3996.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:08:51,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:09:32,447] INFO : Step 40000/50000 rank0 | Train Loss: 7.2145 | Train Loss Unweighted: 2.6885 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3908 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1304 | Regression_eta: 0.0082 | Regression_pt: 0.1331 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7733 | Classification_binary: 2.3908 | Regression_cos_phi: 0.3124 | Regression_energy: 1.4286 | Regression_eta: 0.4011 | Regression_pt: 1.5956 | Regression_sin_phi: 0.3127 | LR: 1.86e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5802s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0120 | max_backward: 0.2033 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5827.7 | uss_MB: 5067.0 | child_rss_MB: 3993.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:09:32,663] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:09:32,683] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 16:09:32,858] INFO : Running validation on rank0 +rank0 [2026-10-28 16:09:32,858] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:09:32,859] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:09:34,227] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 16:09:34,374] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:09:34,508] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:09:34,583] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:09:34,652] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:09:34,720] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:09:34,790] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:09:34,861] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:09:34,973] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:09:35,106] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:09:35,237] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:09:35,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:09:35,379] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:09:35,448] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:09:35,518] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:09:35,589] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:09:35,686] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:09:35,692] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:09:35,695] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:09:35,698] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:09:35,699] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:09:35,702] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:09:35,704] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:09:35,705] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:09:35,708] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:09:35,709] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:09:35,712] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:09:35,714] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:09:35,716] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:09:35,718] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:09:35,721] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:09:35,722] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:09:43,408] INFO : VALIDATION | Step=40000/50000 | Train Loss=7.2145 | Valid Loss=7.4020 | Train Loss Unweighted=2.6885 | Valid Loss Unweighted=2.6488 | Stale=0 | ETA=74.3m +rank0 [2026-10-28 16:09:43,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:43,518] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:09:43,521] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:09:43,697] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:45,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:46,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:47,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:47,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:47,854] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:47,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:47,948] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:09:47,960] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:48,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:50,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:50,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:51,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:52,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:52,273] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:52,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:52,354] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:09:52,357] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:09:52,454] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:54,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:55,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:55,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:56,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:56,844] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:56,852] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:09:57,117] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:09:57,537] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:58,978] INFO : Plotted jets +rank0 [2026-10-28 16:10:00,643] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:00,676] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:10:00,912] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:10:01,303] INFO : Plotted number of elements +rank0 [2026-10-28 16:10:02,704] INFO : Plotted jets +rank0 [2026-10-28 16:10:03,772] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:03,806] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 16:10:04,017] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 16:10:04,378] INFO : Plotted number of elements +rank0 [2026-10-28 16:10:05,764] INFO : Plotted jets +rank0 [2026-10-28 16:10:06,836] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:48,537] INFO : Step 40100/50000 rank0 | Train Loss: 6.6656 | Train Loss Unweighted: 2.6344 | Train Loss Components: Classification: 0.0146 | Classification_binary: 2.3900 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1054 | Regression_eta: 0.0090 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6755 | Classification_binary: 2.3900 | Regression_cos_phi: 0.3995 | Regression_energy: 1.1550 | Regression_eta: 0.4431 | Regression_pt: 1.2487 | Regression_sin_phi: 0.3538 | LR: 1.84e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.7885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.3079 | max_optimizer_step: 0.1472 | max_tensorboard_logging: 0.3770 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6040.8 | uss_MB: 5279.4 | child_rss_MB: 3995.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:10:48,748] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:11:28,582] INFO : Step 40200/50000 rank0 | Train Loss: 7.2354 | Train Loss Unweighted: 2.7217 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4451 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1209 | Regression_eta: 0.0082 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8309 | Classification_binary: 2.4451 | Regression_cos_phi: 0.3822 | Regression_energy: 1.3244 | Regression_eta: 0.4045 | Regression_pt: 1.4045 | Regression_sin_phi: 0.4437 | LR: 1.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6334s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0541 | max_backward: 0.2029 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2991 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6040.8 | uss_MB: 5279.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:11:28,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:09,756] INFO : Step 40300/50000 rank0 | Train Loss: 6.7473 | Train Loss Unweighted: 2.6951 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.4362 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1139 | Regression_eta: 0.0056 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7448 | Classification_binary: 2.4362 | Regression_cos_phi: 0.3621 | Regression_energy: 1.2483 | Regression_eta: 0.2767 | Regression_pt: 1.3557 | Regression_sin_phi: 0.3234 | LR: 1.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0816 | max_forward: 0.0778 | max_loss: 0.0037 | max_backward: 0.3094 | max_optimizer_step: 0.1459 | max_tensorboard_logging: 0.3820 | max_tensorboard_flush: 0.0054 | Memory: rss_MB: 6040.8 | uss_MB: 5279.3 | child_rss_MB: 3989.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:12:09,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:51,090] INFO : Step 40400/50000 rank0 | Train Loss: 6.6618 | Train Loss Unweighted: 2.4133 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.1645 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1071 | Regression_eta: 0.0105 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7980 | Classification_binary: 2.1645 | Regression_cos_phi: 0.4285 | Regression_energy: 1.1740 | Regression_eta: 0.5147 | Regression_pt: 1.2268 | Regression_sin_phi: 0.3552 | LR: 1.79e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8145s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3042 | max_optimizer_step: 0.1480 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6040.8 | uss_MB: 5279.5 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:12:51,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:13:33,322] INFO : Step 40500/50000 rank0 | Train Loss: 7.8518 | Train Loss Unweighted: 2.6107 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2870 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1425 | Regression_eta: 0.0146 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8066 | Classification_binary: 2.2870 | Regression_cos_phi: 0.4167 | Regression_energy: 1.5615 | Regression_eta: 0.7201 | Regression_pt: 1.6369 | Regression_sin_phi: 0.4230 | LR: 1.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7557s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0735 | max_backward: 0.2869 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5899.8 | uss_MB: 5141.4 | child_rss_MB: 3995.8 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:13:33,541] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:15,131] INFO : Step 40600/50000 rank0 | Train Loss: 7.3828 | Train Loss Unweighted: 2.5528 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2621 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1282 | Regression_eta: 0.0134 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7552 | Classification_binary: 2.2621 | Regression_cos_phi: 0.4234 | Regression_energy: 1.4049 | Regression_eta: 0.6572 | Regression_pt: 1.4376 | Regression_sin_phi: 0.4425 | LR: 1.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9108s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1061 | max_loss: 0.0035 | max_backward: 0.3917 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3886 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5899.8 | uss_MB: 5141.5 | child_rss_MB: 3992.3 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:14:15,344] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:55,982] INFO : Step 40700/50000 rank0 | Train Loss: 7.1523 | Train Loss Unweighted: 2.6378 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3613 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1197 | Regression_eta: 0.0082 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7832 | Classification_binary: 2.3613 | Regression_cos_phi: 0.5030 | Regression_energy: 1.3120 | Regression_eta: 0.4035 | Regression_pt: 1.4240 | Regression_sin_phi: 0.3654 | LR: 1.75e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6205s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0657 | max_backward: 0.2030 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5900.8 | uss_MB: 5139.8 | child_rss_MB: 3990.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:14:56,196] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:15:36,607] INFO : Step 40800/50000 rank0 | Train Loss: 6.8422 | Train Loss Unweighted: 2.5608 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3048 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1104 | Regression_eta: 0.0079 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7985 | Classification_binary: 2.3048 | Regression_cos_phi: 0.4799 | Regression_energy: 1.2098 | Regression_eta: 0.3896 | Regression_pt: 1.2931 | Regression_sin_phi: 0.3665 | LR: 1.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5849s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0307 | max_backward: 0.2030 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5900.8 | uss_MB: 5139.8 | child_rss_MB: 3991.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:15:36,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:16:19,105] INFO : Step 40900/50000 rank0 | Train Loss: 8.0948 | Train Loss Unweighted: 2.6801 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3827 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1224 | Regression_eta: 0.0231 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8487 | Classification_binary: 2.3827 | Regression_cos_phi: 0.5323 | Regression_energy: 1.3407 | Regression_eta: 1.1362 | Regression_pt: 1.4324 | Regression_sin_phi: 0.4219 | LR: 1.72e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.8268s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2109 | max_loss: 0.0035 | max_backward: 0.2872 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.2885 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5900.8 | uss_MB: 5141.9 | child_rss_MB: 3993.1 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:16:19,319] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:17:00,475] INFO : Step 41000/50000 rank0 | Train Loss: 7.8388 | Train Loss Unweighted: 2.7823 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4684 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1365 | Regression_eta: 0.0102 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9595 | Classification_binary: 2.4684 | Regression_cos_phi: 0.3974 | Regression_energy: 1.4955 | Regression_eta: 0.5013 | Regression_pt: 1.6131 | Regression_sin_phi: 0.4036 | LR: 1.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7214s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1646 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 5900.8 | uss_MB: 5139.8 | child_rss_MB: 3991.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:17:00,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:17:00,708] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 16:17:00,884] INFO : Running validation on rank0 +rank0 [2026-10-28 16:17:00,884] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:17:00,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:17:02,228] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 16:17:02,376] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:17:02,509] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:17:02,583] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:17:02,652] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:17:02,722] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:17:02,791] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:17:02,861] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:17:02,974] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:17:03,105] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:17:03,237] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:17:03,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:17:03,379] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:17:03,449] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:17:03,519] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:17:03,589] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:17:03,688] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:17:03,695] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:17:03,698] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:17:03,700] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:17:03,701] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:17:03,704] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:17:03,705] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:17:03,708] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:17:03,709] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:17:03,712] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:17:03,713] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:17:03,715] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:17:03,718] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:17:03,719] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:17:03,722] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:17:03,724] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:17:11,452] INFO : VALIDATION | Step=41000/50000 | Train Loss=7.8388 | Valid Loss=7.3869 | Train Loss Unweighted=2.7823 | Valid Loss Unweighted=2.6396 | Stale=0 | ETA=66.9m +rank0 [2026-10-28 16:17:11,462] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:11,548] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:17:11,551] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:17:11,720] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:17:13,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:17:14,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:17:15,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:17:15,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:17:16,113] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:17:16,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:16,180] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:17:16,183] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:17:16,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:17:18,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:17:19,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:17:19,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:17:20,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:17:20,788] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:17:20,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:20,851] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:17:20,854] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:17:20,970] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:17:22,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:17:23,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:17:24,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:17:24,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:17:24,966] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:17:24,974] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:17:25,237] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:17:25,643] INFO : Plotted number of elements +rank0 [2026-10-28 16:17:27,081] INFO : Plotted jets +rank0 [2026-10-28 16:17:28,194] INFO : Plotted jet ratio +rank0 [2026-10-28 16:17:28,228] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:17:28,456] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:17:28,850] INFO : Plotted number of elements +rank0 [2026-10-28 16:17:30,813] INFO : Plotted jets +rank0 [2026-10-28 16:17:31,935] INFO : Plotted jet ratio +rank0 [2026-10-28 16:17:31,968] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 16:17:32,175] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 16:17:32,537] INFO : Plotted number of elements +rank0 [2026-10-28 16:17:33,917] INFO : Plotted jets +rank0 [2026-10-28 16:17:34,990] INFO : Plotted jet ratio +rank0 [2026-10-28 16:18:16,925] INFO : Step 41100/50000 rank0 | Train Loss: 7.3974 | Train Loss Unweighted: 2.5715 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2940 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1187 | Regression_eta: 0.0110 | Regression_pt: 0.1141 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8164 | Classification_binary: 2.2940 | Regression_cos_phi: 0.5815 | Regression_energy: 1.3006 | Regression_eta: 0.5396 | Regression_pt: 1.3683 | Regression_sin_phi: 0.4970 | LR: 1.69e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7516s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1046 | max_loss: 0.0191 | max_backward: 0.2868 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.3128 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6098.1 | uss_MB: 5338.2 | child_rss_MB: 3988.4 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:18:17,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:18:58,069] INFO : Step 41200/50000 rank0 | Train Loss: 8.0287 | Train Loss Unweighted: 2.7188 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3959 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1433 | Regression_eta: 0.0141 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8561 | Classification_binary: 2.3959 | Regression_cos_phi: 0.5258 | Regression_energy: 1.5705 | Regression_eta: 0.6953 | Regression_pt: 1.5997 | Regression_sin_phi: 0.3853 | LR: 1.67e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7461s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.2429 | max_optimizer_step: 0.1460 | max_tensorboard_logging: 0.3902 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6098.1 | uss_MB: 5336.1 | child_rss_MB: 3988.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:18:58,284] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:19:38,325] INFO : Step 41300/50000 rank0 | Train Loss: 7.6282 | Train Loss Unweighted: 2.6651 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3701 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1276 | Regression_eta: 0.0128 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8157 | Classification_binary: 2.3701 | Regression_cos_phi: 0.4844 | Regression_energy: 1.3984 | Regression_eta: 0.6277 | Regression_pt: 1.4745 | Regression_sin_phi: 0.4572 | LR: 1.66e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1830 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.2339 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0128 | Memory: rss_MB: 6098.1 | uss_MB: 5336.1 | child_rss_MB: 3985.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:19:38,540] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:20:20,353] INFO : Step 41400/50000 rank0 | Train Loss: 7.1106 | Train Loss Unweighted: 2.6044 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3307 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1186 | Regression_eta: 0.0105 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7733 | Classification_binary: 2.3307 | Regression_cos_phi: 0.4798 | Regression_energy: 1.2996 | Regression_eta: 0.5146 | Regression_pt: 1.3928 | Regression_sin_phi: 0.3199 | LR: 1.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7403s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1138 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.2970 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 6098.1 | uss_MB: 5336.1 | child_rss_MB: 3982.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:20:20,564] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:21:01,388] INFO : Step 41500/50000 rank0 | Train Loss: 7.2370 | Train Loss Unweighted: 2.6382 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3459 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1293 | Regression_eta: 0.0088 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7981 | Classification_binary: 2.3459 | Regression_cos_phi: 0.3713 | Regression_energy: 1.4165 | Regression_eta: 0.4339 | Regression_pt: 1.5109 | Regression_sin_phi: 0.3604 | LR: 1.63e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7995s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0780 | max_loss: 0.0037 | max_backward: 0.3072 | max_optimizer_step: 0.1453 | max_tensorboard_logging: 0.3813 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5972.3 | uss_MB: 5210.8 | child_rss_MB: 3978.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:21:01,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:21:43,039] INFO : Step 41600/50000 rank0 | Train Loss: 7.3561 | Train Loss Unweighted: 2.6124 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.3155 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1311 | Regression_eta: 0.0084 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7653 | Classification_binary: 2.3155 | Regression_cos_phi: 0.4510 | Regression_energy: 1.4366 | Regression_eta: 0.4118 | Regression_pt: 1.5276 | Regression_sin_phi: 0.4484 | LR: 1.61e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8063s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1800 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2449 | max_tensorboard_logging: 0.2855 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5972.3 | uss_MB: 5210.8 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 16:21:43,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:22:24,694] INFO : Step 41700/50000 rank0 | Train Loss: 7.4663 | Train Loss Unweighted: 2.6716 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3820 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1278 | Regression_eta: 0.0108 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8707 | Classification_binary: 2.3820 | Regression_cos_phi: 0.4541 | Regression_energy: 1.4002 | Regression_eta: 0.5309 | Regression_pt: 1.4333 | Regression_sin_phi: 0.3952 | LR: 1.60e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.0543s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.2134 | max_loss: 0.0036 | max_backward: 0.3916 | max_optimizer_step: 0.1454 | max_tensorboard_logging: 0.4073 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 5972.3 | uss_MB: 5212.9 | child_rss_MB: 3983.8 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:22:24,907] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:06,937] INFO : Step 41800/50000 rank0 | Train Loss: 7.0316 | Train Loss Unweighted: 2.6903 | Train Loss Components: Classification: 0.0150 | Classification_binary: 2.4240 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1169 | Regression_eta: 0.0106 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6943 | Classification_binary: 2.4240 | Regression_cos_phi: 0.3705 | Regression_energy: 1.2812 | Regression_eta: 0.5227 | Regression_pt: 1.3473 | Regression_sin_phi: 0.3916 | LR: 1.58e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.8700s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0419 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.3100 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.4098 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 5972.3 | uss_MB: 5211.0 | child_rss_MB: 3988.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 16:23:07,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:47,731] INFO : Step 41900/50000 rank0 | Train Loss: 6.9989 | Train Loss Unweighted: 2.6817 | Train Loss Components: Classification: 0.0150 | Classification_binary: 2.4174 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1164 | Regression_eta: 0.0085 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6971 | Classification_binary: 2.4174 | Regression_cos_phi: 0.4146 | Regression_energy: 1.2749 | Regression_eta: 0.4176 | Regression_pt: 1.3374 | Regression_sin_phi: 0.4399 | LR: 1.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6871s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0890 | max_backward: 0.2037 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.3031 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5972.3 | uss_MB: 5210.9 | child_rss_MB: 3988.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:23:47,945] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:24:28,385] INFO : Step 42000/50000 rank0 | Train Loss: 7.3358 | Train Loss Unweighted: 2.6308 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3459 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1245 | Regression_eta: 0.0112 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7883 | Classification_binary: 2.3459 | Regression_cos_phi: 0.4337 | Regression_energy: 1.3638 | Regression_eta: 0.5498 | Regression_pt: 1.4327 | Regression_sin_phi: 0.4217 | LR: 1.56e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6265s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0284 | max_backward: 0.2033 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3128 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5972.3 | uss_MB: 5211.0 | child_rss_MB: 3987.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:24:28,599] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:24:28,619] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 16:24:28,796] INFO : Running validation on rank0 +rank0 [2026-10-28 16:24:28,796] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:24:28,796] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:24:30,165] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 16:24:30,312] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:24:30,445] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:24:30,518] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:24:30,588] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:24:30,657] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:24:30,727] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:24:30,797] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:24:30,910] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:24:31,043] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:24:31,178] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:24:31,249] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:24:31,320] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:24:31,388] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:24:31,458] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:24:31,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:24:31,629] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:24:31,636] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:24:31,639] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:24:31,642] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:24:31,643] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:24:31,645] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:24:31,646] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:24:31,649] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:24:31,652] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:24:31,653] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:24:31,655] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:24:31,656] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:24:31,659] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:24:31,662] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:24:31,664] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:24:31,666] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:24:39,361] INFO : VALIDATION | Step=42000/50000 | Train Loss=7.3358 | Valid Loss=7.3656 | Train Loss Unweighted=2.6308 | Valid Loss Unweighted=2.6342 | Stale=0 | ETA=59.4m +rank0 [2026-10-28 16:24:39,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:24:39,450] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:24:39,454] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:24:39,634] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:24:41,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:24:42,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:24:42,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:24:43,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:24:43,771] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:24:43,774] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:24:43,847] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:24:43,850] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:24:43,955] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:24:45,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:24:46,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:24:47,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:24:47,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:24:47,970] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:24:47,973] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:24:48,041] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:24:48,044] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:24:48,152] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:24:50,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:24:50,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:24:51,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:24:51,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:24:52,158] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:24:52,165] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:24:52,428] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:24:52,834] INFO : Plotted number of elements +rank0 [2026-10-28 16:24:54,280] INFO : Plotted jets +rank0 [2026-10-28 16:24:56,011] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:56,044] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:24:56,279] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:24:56,671] INFO : Plotted number of elements +rank0 [2026-10-28 16:24:58,064] INFO : Plotted jets +rank0 [2026-10-28 16:24:59,191] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:59,224] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 16:24:59,433] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 16:24:59,799] INFO : Plotted number of elements +rank0 [2026-10-28 16:25:01,176] INFO : Plotted jets +rank0 [2026-10-28 16:25:02,244] INFO : Plotted jet ratio +rank0 [2026-10-28 16:25:44,380] INFO : Step 42100/50000 rank0 | Train Loss: 6.9453 | Train Loss Unweighted: 2.6282 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.3685 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1133 | Regression_eta: 0.0085 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7616 | Classification_binary: 2.3685 | Regression_cos_phi: 0.5189 | Regression_energy: 1.2419 | Regression_eta: 0.4178 | Regression_pt: 1.3042 | Regression_sin_phi: 0.3324 | LR: 1.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7066s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.3063 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6176.7 | uss_MB: 5415.6 | child_rss_MB: 3989.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 16:25:44,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:26:25,793] INFO : Step 42200/50000 rank0 | Train Loss: 7.4732 | Train Loss Unweighted: 2.6061 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3164 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1238 | Regression_eta: 0.0129 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8144 | Classification_binary: 2.3164 | Regression_cos_phi: 0.3915 | Regression_energy: 1.3566 | Regression_eta: 0.6340 | Regression_pt: 1.4630 | Regression_sin_phi: 0.4972 | LR: 1.53e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8470s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1856 | max_loss: 0.0038 | max_backward: 0.2031 | max_optimizer_step: 0.2513 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6176.7 | uss_MB: 5415.6 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:26:26,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:26:35,757] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 16:26:35,757] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 16:27:06,730] INFO : Step 42300/50000 rank0 | Train Loss: 6.6740 | Train Loss Unweighted: 2.5505 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3003 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1093 | Regression_eta: 0.0080 | Regression_pt: 0.1050 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7681 | Classification_binary: 2.3003 | Regression_cos_phi: 0.3820 | Regression_energy: 1.1979 | Regression_eta: 0.3937 | Regression_pt: 1.2592 | Regression_sin_phi: 0.3728 | LR: 1.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9014s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0875 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3066 | max_optimizer_step: 0.1487 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 6214.8 | uss_MB: 5453.5 | child_rss_MB: 3994.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:27:06,941] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:27:48,525] INFO : Step 42400/50000 rank0 | Train Loss: 7.4859 | Train Loss Unweighted: 2.6139 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3359 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1193 | Regression_eta: 0.0176 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8328 | Classification_binary: 2.3359 | Regression_cos_phi: 0.4099 | Regression_energy: 1.3071 | Regression_eta: 0.8676 | Regression_pt: 1.3312 | Regression_sin_phi: 0.4015 | LR: 1.50e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8480s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1049 | max_loss: 0.0929 | max_backward: 0.2872 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6214.8 | uss_MB: 5455.6 | child_rss_MB: 3996.8 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 90.9 +rank0 [2026-10-28 16:27:48,757] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:28:29,597] INFO : Step 42500/50000 rank0 | Train Loss: 7.1315 | Train Loss Unweighted: 2.6342 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3610 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1190 | Regression_eta: 0.0089 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7851 | Classification_binary: 2.3610 | Regression_cos_phi: 0.4659 | Regression_energy: 1.3042 | Regression_eta: 0.4400 | Regression_pt: 1.3852 | Regression_sin_phi: 0.3901 | LR: 1.49e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8788s | Phase Times: max_pre_step_barrier: 0.1056 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0036 | max_backward: 0.3088 | max_optimizer_step: 0.1432 | max_tensorboard_logging: 0.3728 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6041.4 | uss_MB: 5279.8 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:28:29,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:10,605] INFO : Step 42600/50000 rank0 | Train Loss: 7.4452 | Train Loss Unweighted: 2.6580 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3860 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1149 | Regression_eta: 0.0147 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7850 | Classification_binary: 2.3860 | Regression_cos_phi: 0.4471 | Regression_energy: 1.2591 | Regression_eta: 0.7216 | Regression_pt: 1.3316 | Regression_sin_phi: 0.5149 | LR: 1.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2104 | max_loss: 0.0039 | max_backward: 0.2871 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6042.4 | uss_MB: 5282.3 | child_rss_MB: 3992.8 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:29:10,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:51,763] INFO : Step 42700/50000 rank0 | Train Loss: 7.2887 | Train Loss Unweighted: 2.6532 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.3642 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1265 | Regression_eta: 0.0094 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7465 | Classification_binary: 2.3642 | Regression_cos_phi: 0.4577 | Regression_energy: 1.3863 | Regression_eta: 0.4601 | Regression_pt: 1.4932 | Regression_sin_phi: 0.3806 | LR: 1.47e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6965s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0038 | max_backward: 0.2034 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.3874 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6042.4 | uss_MB: 5280.3 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:29:51,974] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:30:32,901] INFO : Step 42800/50000 rank0 | Train Loss: 6.9594 | Train Loss Unweighted: 2.6308 | Train Loss Components: Classification: 0.0155 | Classification_binary: 2.3664 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1170 | Regression_eta: 0.0100 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7169 | Classification_binary: 2.3664 | Regression_cos_phi: 0.3894 | Regression_energy: 1.2815 | Regression_eta: 0.4923 | Regression_pt: 1.3218 | Regression_sin_phi: 0.3911 | LR: 1.45e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7803s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0159 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3086 | max_optimizer_step: 0.1455 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6042.4 | uss_MB: 5280.4 | child_rss_MB: 3995.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:30:33,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:31:14,604] INFO : Step 42900/50000 rank0 | Train Loss: 6.7383 | Train Loss Unweighted: 2.6087 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.3536 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1105 | Regression_eta: 0.0079 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7526 | Classification_binary: 2.3536 | Regression_cos_phi: 0.3726 | Regression_energy: 1.2112 | Regression_eta: 0.3893 | Regression_pt: 1.3153 | Regression_sin_phi: 0.3438 | LR: 1.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7888s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0116 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.3104 | max_optimizer_step: 0.1450 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0076 | Memory: rss_MB: 6042.4 | uss_MB: 5280.0 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:31:14,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:31:55,741] INFO : Step 43000/50000 rank0 | Train Loss: 7.3513 | Train Loss Unweighted: 2.6736 | Train Loss Components: Classification: 0.0155 | Classification_binary: 2.3915 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1228 | Regression_eta: 0.0100 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7168 | Classification_binary: 2.3915 | Regression_cos_phi: 0.4890 | Regression_energy: 1.3450 | Regression_eta: 0.4929 | Regression_pt: 1.4295 | Regression_sin_phi: 0.4865 | LR: 1.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1848 | max_loss: 0.0036 | max_backward: 0.2029 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.2856 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6042.4 | uss_MB: 5280.3 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:31:55,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:31:55,984] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 16:31:56,158] INFO : Running validation on rank0 +rank0 [2026-10-28 16:31:56,158] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:31:56,158] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:31:57,520] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 16:31:57,667] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:31:57,801] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:31:57,875] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:31:57,945] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:31:58,014] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:31:58,083] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:31:58,152] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:31:58,265] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:31:58,397] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:31:58,529] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:31:58,601] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:31:58,671] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:31:58,739] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:31:58,808] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:31:58,878] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:31:58,978] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:31:58,984] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:31:58,986] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:31:58,989] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:31:58,990] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:31:58,993] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:31:58,994] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:31:58,997] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:31:58,999] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:31:59,000] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:31:59,003] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:31:59,004] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:31:59,007] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:31:59,009] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:31:59,010] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:31:59,013] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:32:06,726] INFO : VALIDATION | Step=43000/50000 | Train Loss=7.3513 | Valid Loss=7.3589 | Train Loss Unweighted=2.6736 | Valid Loss Unweighted=2.6217 | Stale=0 | ETA=52.0m +rank0 [2026-10-28 16:32:06,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:32:06,860] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:32:06,863] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:32:07,071] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:32:09,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:32:09,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:32:10,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:32:11,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:32:11,331] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:32:11,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:32:11,431] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:32:11,440] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:32:11,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:32:13,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:32:14,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:32:15,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:32:15,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:32:16,021] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:32:16,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:32:16,172] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:32:16,174] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:32:16,363] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:32:18,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:32:18,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:32:19,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:32:20,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:32:20,353] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:32:20,361] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:32:20,622] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:32:21,028] INFO : Plotted number of elements +rank0 [2026-10-28 16:32:22,467] INFO : Plotted jets +rank0 [2026-10-28 16:32:23,562] INFO : Plotted jet ratio +rank0 [2026-10-28 16:32:23,594] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:32:23,822] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:32:24,211] INFO : Plotted number of elements +rank0 [2026-10-28 16:32:26,228] INFO : Plotted jets +rank0 [2026-10-28 16:32:27,349] INFO : Plotted jet ratio +rank0 [2026-10-28 16:32:27,382] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 16:32:27,590] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 16:32:27,952] INFO : Plotted number of elements +rank0 [2026-10-28 16:32:29,331] INFO : Plotted jets +rank0 [2026-10-28 16:32:30,396] INFO : Plotted jet ratio +rank0 [2026-10-28 16:33:12,169] INFO : Step 43100/50000 rank0 | Train Loss: 7.0543 | Train Loss Unweighted: 2.5818 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.3174 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1143 | Regression_eta: 0.0117 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7313 | Classification_binary: 2.3174 | Regression_cos_phi: 0.4519 | Regression_energy: 1.2528 | Regression_eta: 0.5733 | Regression_pt: 1.3158 | Regression_sin_phi: 0.4119 | LR: 1.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6369s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0634 | max_backward: 0.2031 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6243.8 | uss_MB: 5482.6 | child_rss_MB: 3993.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:33:12,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:33:53,463] INFO : Step 43200/50000 rank0 | Train Loss: 7.1700 | Train Loss Unweighted: 2.5837 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3139 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1170 | Regression_eta: 0.0117 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7696 | Classification_binary: 2.3139 | Regression_cos_phi: 0.4726 | Regression_energy: 1.2825 | Regression_eta: 0.5754 | Regression_pt: 1.3312 | Regression_sin_phi: 0.4248 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9081s | Phase Times: max_pre_step_barrier: 0.1050 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3110 | max_optimizer_step: 0.1498 | max_tensorboard_logging: 0.3779 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6243.8 | uss_MB: 5482.6 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:33:53,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:34:35,081] INFO : Step 43300/50000 rank0 | Train Loss: 7.2155 | Train Loss Unweighted: 2.5591 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.2859 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1177 | Regression_eta: 0.0118 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7840 | Classification_binary: 2.2859 | Regression_cos_phi: 0.4922 | Regression_energy: 1.2893 | Regression_eta: 0.5786 | Regression_pt: 1.3566 | Regression_sin_phi: 0.4289 | LR: 1.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8022s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2097 | max_loss: 0.0037 | max_backward: 0.2869 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6243.8 | uss_MB: 5484.7 | child_rss_MB: 3989.3 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:34:35,294] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:16,482] INFO : Step 43400/50000 rank0 | Train Loss: 7.4231 | Train Loss Unweighted: 2.6635 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3845 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1221 | Regression_eta: 0.0113 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8227 | Classification_binary: 2.3845 | Regression_cos_phi: 0.5108 | Regression_energy: 1.3383 | Regression_eta: 0.5554 | Regression_pt: 1.3610 | Regression_sin_phi: 0.4504 | LR: 1.38e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8321s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2100 | max_loss: 0.0037 | max_backward: 0.2868 | max_optimizer_step: 0.1522 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6243.8 | uss_MB: 5484.8 | child_rss_MB: 3994.3 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 16:35:16,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:57,600] INFO : Step 43500/50000 rank0 | Train Loss: 7.8412 | Train Loss Unweighted: 2.6222 | Train Loss Components: Classification: 0.0141 | Classification_binary: 2.3187 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1333 | Regression_eta: 0.0215 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6560 | Classification_binary: 2.3187 | Regression_cos_phi: 0.4751 | Regression_energy: 1.4611 | Regression_eta: 1.0566 | Regression_pt: 1.4527 | Regression_sin_phi: 0.4210 | LR: 1.37e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8675s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0018 | max_forward: 0.1048 | max_loss: 0.0036 | max_backward: 0.3941 | max_optimizer_step: 0.1462 | max_tensorboard_logging: 0.3431 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6121.1 | uss_MB: 5363.6 | child_rss_MB: 3994.6 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 16:35:57,814] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:36:39,508] INFO : Step 43600/50000 rank0 | Train Loss: 8.0592 | Train Loss Unweighted: 2.7722 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.4514 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1422 | Regression_eta: 0.0170 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7677 | Classification_binary: 2.4514 | Regression_cos_phi: 0.4877 | Regression_energy: 1.5578 | Regression_eta: 0.8374 | Regression_pt: 1.5847 | Regression_sin_phi: 0.3725 | LR: 1.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9042s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0969 | max_forward: 0.0774 | max_loss: 0.0036 | max_backward: 0.3084 | max_optimizer_step: 0.1482 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6121.1 | uss_MB: 5362.3 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:36:39,721] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:20,319] INFO : Step 43700/50000 rank0 | Train Loss: 7.3123 | Train Loss Unweighted: 2.6771 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.3853 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1302 | Regression_eta: 0.0092 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7511 | Classification_binary: 2.3853 | Regression_cos_phi: 0.4061 | Regression_energy: 1.4261 | Regression_eta: 0.4537 | Regression_pt: 1.4886 | Regression_sin_phi: 0.4015 | LR: 1.35e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7694s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2862 | max_optimizer_step: 0.1455 | max_tensorboard_logging: 0.3885 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6121.1 | uss_MB: 5362.5 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:37:20,533] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:38:02,240] INFO : Step 43800/50000 rank0 | Train Loss: 7.3878 | Train Loss Unweighted: 2.6173 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3321 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1250 | Regression_eta: 0.0118 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8046 | Classification_binary: 2.3321 | Regression_cos_phi: 0.4696 | Regression_energy: 1.3700 | Regression_eta: 0.5814 | Regression_pt: 1.4118 | Regression_sin_phi: 0.4185 | LR: 1.34e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7305s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1666 | max_loss: 0.0035 | max_backward: 0.2038 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.2871 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6121.1 | uss_MB: 5362.5 | child_rss_MB: 3992.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:38:02,453] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:38:43,291] INFO : Step 43900/50000 rank0 | Train Loss: 7.7212 | Train Loss Unweighted: 2.6081 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3048 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1333 | Regression_eta: 0.0147 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7781 | Classification_binary: 2.3048 | Regression_cos_phi: 0.5628 | Regression_energy: 1.4601 | Regression_eta: 0.7234 | Regression_pt: 1.4897 | Regression_sin_phi: 0.4024 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7715s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.2997 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6121.1 | uss_MB: 5362.6 | child_rss_MB: 3991.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:38:43,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:39:24,914] INFO : Step 44000/50000 rank0 | Train Loss: 7.9835 | Train Loss Unweighted: 2.6233 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.3119 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1309 | Regression_eta: 0.0240 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6935 | Classification_binary: 2.3119 | Regression_cos_phi: 0.3953 | Regression_energy: 1.4346 | Regression_eta: 1.1799 | Regression_pt: 1.5506 | Regression_sin_phi: 0.4177 | LR: 1.32e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7718s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1854 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2233 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6121.1 | uss_MB: 5362.6 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:39:25,131] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:39:25,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 16:39:25,327] INFO : Running validation on rank0 +rank0 [2026-10-28 16:39:25,327] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:39:25,327] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:39:26,668] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 16:39:26,815] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:39:26,948] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:39:27,023] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:39:27,093] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:39:27,179] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:39:27,251] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:39:27,322] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:39:27,451] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:39:27,584] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:39:27,716] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:39:27,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:39:27,860] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:39:27,929] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:39:27,998] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:39:28,069] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:39:28,169] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:39:28,177] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:39:28,179] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:39:28,181] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:39:28,183] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:39:28,186] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:39:28,187] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:39:28,190] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:39:28,192] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:39:28,193] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:39:28,196] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:39:28,197] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:39:28,200] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:39:28,202] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:39:28,205] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:39:28,206] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:39:35,837] INFO : VALIDATION | Step=44000/50000 | Train Loss=7.9835 | Valid Loss=7.3530 | Train Loss Unweighted=2.6233 | Valid Loss Unweighted=2.6168 | Stale=0 | ETA=44.6m +rank0 [2026-10-28 16:39:35,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:39:35,905] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:39:35,908] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:39:36,026] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:39:38,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:39:38,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:39:39,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:39:40,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:39:40,449] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:39:40,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:39:40,530] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:39:40,533] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:39:40,633] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:39:42,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:39:43,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:39:43,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:39:44,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:39:44,701] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:39:44,704] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:39:44,760] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:39:44,763] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:39:44,887] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:39:46,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:39:47,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:39:48,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:39:48,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:39:48,871] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:39:48,880] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:39:49,136] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:39:49,544] INFO : Plotted number of elements +rank0 [2026-10-28 16:39:50,986] INFO : Plotted jets +rank0 [2026-10-28 16:39:52,079] INFO : Plotted jet ratio +rank0 [2026-10-28 16:39:52,112] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:39:52,338] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:39:52,730] INFO : Plotted number of elements +rank0 [2026-10-28 16:39:54,719] INFO : Plotted jets +rank0 [2026-10-28 16:39:55,849] INFO : Plotted jet ratio +rank0 [2026-10-28 16:39:55,882] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 16:39:56,088] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 16:39:56,449] INFO : Plotted number of elements +rank0 [2026-10-28 16:39:57,831] INFO : Plotted jets +rank0 [2026-10-28 16:39:58,907] INFO : Plotted jet ratio +rank0 [2026-10-28 16:40:40,606] INFO : Step 44100/50000 rank0 | Train Loss: 7.8008 | Train Loss Unweighted: 2.7684 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4718 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1297 | Regression_eta: 0.0175 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8000 | Classification_binary: 2.4718 | Regression_cos_phi: 0.4357 | Regression_energy: 1.4214 | Regression_eta: 0.8624 | Regression_pt: 1.4397 | Regression_sin_phi: 0.3698 | LR: 1.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7865s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.2811 | max_optimizer_step: 0.1491 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:40:40,820] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:41:22,599] INFO : Step 44200/50000 rank0 | Train Loss: 7.8458 | Train Loss Unweighted: 2.6598 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3615 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1285 | Regression_eta: 0.0210 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8186 | Classification_binary: 2.3615 | Regression_cos_phi: 0.4403 | Regression_energy: 1.4079 | Regression_eta: 1.0323 | Regression_pt: 1.4306 | Regression_sin_phi: 0.3545 | LR: 1.30e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.7807s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.2768 | max_optimizer_step: 0.1503 | max_tensorboard_logging: 0.3787 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:41:22,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:42:03,794] INFO : Step 44300/50000 rank0 | Train Loss: 6.8814 | Train Loss Unweighted: 2.5129 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2563 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1104 | Regression_eta: 0.0108 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7753 | Classification_binary: 2.2563 | Regression_cos_phi: 0.3963 | Regression_energy: 1.2101 | Regression_eta: 0.5285 | Regression_pt: 1.2718 | Regression_sin_phi: 0.4431 | LR: 1.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8954s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0708 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3087 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 0.3916 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:42:04,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:42:45,090] INFO : Step 44400/50000 rank0 | Train Loss: 7.2231 | Train Loss Unweighted: 2.5734 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2772 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1309 | Regression_eta: 0.0077 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7925 | Classification_binary: 2.2772 | Regression_cos_phi: 0.4492 | Regression_energy: 1.4347 | Regression_eta: 0.3788 | Regression_pt: 1.5420 | Regression_sin_phi: 0.3487 | LR: 1.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8540s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0446 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.3107 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.3819 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3995.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:42:45,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:43:26,178] INFO : Step 44500/50000 rank0 | Train Loss: 7.4555 | Train Loss Unweighted: 2.6195 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3398 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1207 | Regression_eta: 0.0141 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8442 | Classification_binary: 2.3398 | Regression_cos_phi: 0.4402 | Regression_energy: 1.3223 | Regression_eta: 0.6925 | Regression_pt: 1.3578 | Regression_sin_phi: 0.4586 | LR: 1.27e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6780s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0801 | max_loss: 0.0034 | max_backward: 0.2034 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.3070 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6197.9 | uss_MB: 5437.1 | child_rss_MB: 3991.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:43:26,393] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:44:06,829] INFO : Step 44600/50000 rank0 | Train Loss: 7.8700 | Train Loss Unweighted: 2.5389 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2161 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1377 | Regression_eta: 0.0185 | Regression_pt: 0.1374 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8141 | Classification_binary: 2.2161 | Regression_cos_phi: 0.3750 | Regression_energy: 1.5089 | Regression_eta: 0.9113 | Regression_pt: 1.6474 | Regression_sin_phi: 0.3973 | LR: 1.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8312s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1833 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.3181 | max_tensorboard_flush: 0.0157 | Memory: rss_MB: 6199.9 | uss_MB: 5438.6 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:44:07,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:44:48,611] INFO : Step 44700/50000 rank0 | Train Loss: 7.3791 | Train Loss Unweighted: 2.5887 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3021 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1238 | Regression_eta: 0.0128 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7816 | Classification_binary: 2.3021 | Regression_cos_phi: 0.4705 | Regression_energy: 1.3565 | Regression_eta: 0.6280 | Regression_pt: 1.4413 | Regression_sin_phi: 0.3991 | LR: 1.25e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7067s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0769 | max_backward: 0.2037 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.3072 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6199.9 | uss_MB: 5438.8 | child_rss_MB: 3998.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:44:48,826] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:45:29,934] INFO : Step 44800/50000 rank0 | Train Loss: 7.1036 | Train Loss Unweighted: 2.5417 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.2537 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1274 | Regression_eta: 0.0094 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7683 | Classification_binary: 2.2537 | Regression_cos_phi: 0.3839 | Regression_energy: 1.3962 | Regression_eta: 0.4614 | Regression_pt: 1.4813 | Regression_sin_phi: 0.3587 | LR: 1.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8725s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0018 | max_forward: 0.1041 | max_loss: 0.0034 | max_backward: 0.3932 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.3475 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 6199.9 | uss_MB: 5441.3 | child_rss_MB: 3998.0 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:45:30,147] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:46:11,254] INFO : Step 44900/50000 rank0 | Train Loss: 7.3895 | Train Loss Unweighted: 2.6300 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3597 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1152 | Regression_eta: 0.0115 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7903 | Classification_binary: 2.3597 | Regression_cos_phi: 0.6397 | Regression_energy: 1.2618 | Regression_eta: 0.5667 | Regression_pt: 1.3237 | Regression_sin_phi: 0.4475 | LR: 1.23e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6999s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1334 | max_loss: 0.0034 | max_backward: 0.2035 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6199.9 | uss_MB: 5439.3 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:46:11,470] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:46:52,451] INFO : Step 45000/50000 rank0 | Train Loss: 7.2287 | Train Loss Unweighted: 2.7190 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.4407 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1209 | Regression_eta: 0.0098 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7559 | Classification_binary: 2.4407 | Regression_cos_phi: 0.4100 | Regression_energy: 1.3244 | Regression_eta: 0.4811 | Regression_pt: 1.4315 | Regression_sin_phi: 0.3850 | LR: 1.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6841s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0741 | max_backward: 0.2037 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6199.9 | uss_MB: 5439.4 | child_rss_MB: 3996.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:46:52,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:46:52,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 16:46:52,862] INFO : Running validation on rank0 +rank0 [2026-10-28 16:46:52,862] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:46:52,862] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:46:54,195] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 16:46:54,342] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:46:54,473] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:46:54,547] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:46:54,617] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:46:54,686] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:46:54,756] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:46:54,826] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:46:54,940] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:46:55,071] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:46:55,203] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:46:55,275] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:46:55,346] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:46:55,414] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:46:55,483] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:46:55,553] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:46:55,650] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:46:55,656] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:46:55,659] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:46:55,662] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:46:55,663] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:46:55,673] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:46:55,680] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:46:55,683] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:46:55,686] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:46:55,687] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:46:55,689] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:46:55,692] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:46:55,693] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:46:55,696] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:46:55,697] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:46:55,699] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:47:03,450] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.2287 | Valid Loss=7.3374 | Train Loss Unweighted=2.7190 | Valid Loss Unweighted=2.6159 | Stale=0 | ETA=37.2m +rank0 [2026-10-28 16:47:03,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:47:03,517] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:47:03,520] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:47:03,702] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:47:05,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:47:06,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:47:07,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:47:07,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:47:08,022] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:47:08,025] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:47:08,081] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:47:08,084] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:47:08,199] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:47:10,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:47:10,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:47:11,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:47:12,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:47:12,351] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:47:12,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:47:12,410] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:47:12,413] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:47:12,531] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:47:14,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:47:15,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:47:15,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:47:16,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:47:16,431] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:47:16,439] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:47:16,698] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:47:17,111] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:18,562] INFO : Plotted jets +rank0 [2026-10-28 16:47:19,671] INFO : Plotted jet ratio +rank0 [2026-10-28 16:47:19,705] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:47:19,933] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:47:20,325] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:21,716] INFO : Plotted jets +rank0 [2026-10-28 16:47:23,396] INFO : Plotted jet ratio +rank0 [2026-10-28 16:47:23,429] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 16:47:23,637] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 16:47:24,002] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:25,378] INFO : Plotted jets +rank0 [2026-10-28 16:47:26,440] INFO : Plotted jet ratio +rank0 [2026-10-28 16:48:07,596] INFO : Step 45100/50000 rank0 | Train Loss: 7.6104 | Train Loss Unweighted: 2.4881 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.1962 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1255 | Regression_eta: 0.0154 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8595 | Classification_binary: 2.1962 | Regression_cos_phi: 0.5865 | Regression_energy: 1.3752 | Regression_eta: 0.7547 | Regression_pt: 1.4078 | Regression_sin_phi: 0.4304 | LR: 1.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7954s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1041 | max_loss: 0.0738 | max_backward: 0.2874 | max_optimizer_step: 0.1491 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6405.8 | uss_MB: 5647.5 | child_rss_MB: 3990.5 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:48:07,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:48:48,388] INFO : Step 45200/50000 rank0 | Train Loss: 7.8298 | Train Loss Unweighted: 2.7218 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4131 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1338 | Regression_eta: 0.0134 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7794 | Classification_binary: 2.4131 | Regression_cos_phi: 0.4656 | Regression_energy: 1.4664 | Regression_eta: 0.6589 | Regression_pt: 1.5644 | Regression_sin_phi: 0.4819 | LR: 1.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6787s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.2959 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6405.8 | uss_MB: 5645.4 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:48:48,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:49:30,157] INFO : Step 45300/50000 rank0 | Train Loss: 7.2037 | Train Loss Unweighted: 2.7056 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4312 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1213 | Regression_eta: 0.0075 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8098 | Classification_binary: 2.4312 | Regression_cos_phi: 0.4800 | Regression_energy: 1.3289 | Regression_eta: 0.3693 | Regression_pt: 1.3784 | Regression_sin_phi: 0.4060 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7792s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.2886 | max_optimizer_step: 0.1465 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6405.8 | uss_MB: 5645.5 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:49:30,372] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:50:11,263] INFO : Step 45400/50000 rank0 | Train Loss: 6.7664 | Train Loss Unweighted: 2.5806 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.3229 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1131 | Regression_eta: 0.0084 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7317 | Classification_binary: 2.3229 | Regression_cos_phi: 0.4097 | Regression_energy: 1.2392 | Regression_eta: 0.4134 | Regression_pt: 1.3098 | Regression_sin_phi: 0.3397 | LR: 1.19e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8022s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1554 | max_loss: 0.0037 | max_backward: 0.2030 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.3025 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 6405.8 | uss_MB: 5645.1 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:50:11,479] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:50:52,271] INFO : Step 45500/50000 rank0 | Train Loss: 6.9032 | Train Loss Unweighted: 2.5501 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2815 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1157 | Regression_eta: 0.0082 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7748 | Classification_binary: 2.2815 | Regression_cos_phi: 0.3871 | Regression_energy: 1.2676 | Regression_eta: 0.4013 | Regression_pt: 1.3932 | Regression_sin_phi: 0.3976 | LR: 1.18e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.9213s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0409 | max_forward: 0.0773 | max_loss: 0.0036 | max_backward: 0.3101 | max_optimizer_step: 0.1512 | max_tensorboard_logging: 0.4153 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6271.8 | uss_MB: 5510.5 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:50:52,485] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:51:34,302] INFO : Step 45600/50000 rank0 | Train Loss: 7.5787 | Train Loss Unweighted: 2.5387 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2300 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1354 | Regression_eta: 0.0121 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8265 | Classification_binary: 2.2300 | Regression_cos_phi: 0.4710 | Regression_energy: 1.4833 | Regression_eta: 0.5928 | Regression_pt: 1.5625 | Regression_sin_phi: 0.4127 | LR: 1.17e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0411 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3100 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.3953 | max_tensorboard_flush: 0.0196 | Memory: rss_MB: 6272.8 | uss_MB: 5511.6 | child_rss_MB: 3996.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:51:34,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:15,392] INFO : Step 45700/50000 rank0 | Train Loss: 7.4985 | Train Loss Unweighted: 2.6312 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3257 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1352 | Regression_eta: 0.0098 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7759 | Classification_binary: 2.3257 | Regression_cos_phi: 0.4876 | Regression_energy: 1.4815 | Regression_eta: 0.4837 | Regression_pt: 1.5687 | Regression_sin_phi: 0.3754 | LR: 1.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7997s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1336 | max_loss: 0.0037 | max_backward: 0.2032 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.3454 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6272.8 | uss_MB: 5511.8 | child_rss_MB: 4014.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:52:15,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:56,398] INFO : Step 45800/50000 rank0 | Train Loss: 7.7721 | Train Loss Unweighted: 2.6494 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3672 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1177 | Regression_eta: 0.0143 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8149 | Classification_binary: 2.3672 | Regression_cos_phi: 0.6675 | Regression_energy: 1.2899 | Regression_eta: 0.7037 | Regression_pt: 1.3697 | Regression_sin_phi: 0.5591 | LR: 1.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2111 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0123 | Memory: rss_MB: 6272.8 | uss_MB: 5512.0 | child_rss_MB: 3999.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:52:56,611] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:53:38,341] INFO : Step 45900/50000 rank0 | Train Loss: 7.5350 | Train Loss Unweighted: 2.7039 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4147 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1249 | Regression_eta: 0.0113 | Regression_pt: 0.1216 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8244 | Classification_binary: 2.4147 | Regression_cos_phi: 0.4713 | Regression_energy: 1.3685 | Regression_eta: 0.5560 | Regression_pt: 1.4579 | Regression_sin_phi: 0.4422 | LR: 1.15e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7208s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1491 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.2771 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6272.8 | uss_MB: 5512.4 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:53:38,555] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:54:20,371] INFO : Step 46000/50000 rank0 | Train Loss: 7.7253 | Train Loss Unweighted: 2.6006 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3084 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1257 | Regression_eta: 0.0179 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7771 | Classification_binary: 2.3084 | Regression_cos_phi: 0.5385 | Regression_energy: 1.3777 | Regression_eta: 0.8818 | Regression_pt: 1.4056 | Regression_sin_phi: 0.4362 | LR: 1.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7877s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1858 | max_loss: 0.0037 | max_backward: 0.2031 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6272.8 | uss_MB: 5512.4 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:54:20,586] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:54:20,606] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 16:54:20,779] INFO : Running validation on rank0 +rank0 [2026-10-28 16:54:20,779] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:54:20,779] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:54:22,102] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 16:54:22,249] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:54:22,382] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:54:22,456] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:54:22,525] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:54:22,595] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:54:22,665] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:54:22,734] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:54:22,847] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:54:22,979] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:54:23,112] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:54:23,184] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:54:23,253] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:54:23,322] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:54:23,393] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:54:23,463] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:54:23,562] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:54:23,568] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:54:23,571] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:54:23,574] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:54:23,577] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:54:23,578] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:54:23,581] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:54:23,582] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:54:23,584] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:54:23,587] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:54:23,590] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:54:23,591] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:54:23,594] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:54:23,597] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:54:23,599] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:54:23,600] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:54:31,310] INFO : VALIDATION | Step=46000/50000 | Train Loss=7.7253 | Valid Loss=7.3077 | Train Loss Unweighted=2.6006 | Valid Loss Unweighted=2.6065 | Stale=0 | ETA=29.7m +rank0 [2026-10-28 16:54:31,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:54:31,401] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:54:31,405] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:54:31,509] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:54:33,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:54:34,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:54:34,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:54:35,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:54:35,737] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:54:35,740] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:54:35,845] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:54:35,848] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:54:36,037] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:54:38,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:54:38,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:54:39,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:54:39,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:54:40,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:54:40,099] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:54:40,188] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:54:40,208] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:54:40,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:54:42,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:54:42,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:54:43,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:54:44,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:54:44,289] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:54:44,296] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:54:44,555] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:54:44,961] INFO : Plotted number of elements +rank0 [2026-10-28 16:54:46,398] INFO : Plotted jets +rank0 [2026-10-28 16:54:47,510] INFO : Plotted jet ratio +rank0 [2026-10-28 16:54:47,544] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:54:47,774] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:54:48,163] INFO : Plotted number of elements +rank0 [2026-10-28 16:54:49,549] INFO : Plotted jets +rank0 [2026-10-28 16:54:51,230] INFO : Plotted jet ratio +rank0 [2026-10-28 16:54:51,266] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 16:54:51,473] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 16:54:51,833] INFO : Plotted number of elements +rank0 [2026-10-28 16:54:53,204] INFO : Plotted jets +rank0 [2026-10-28 16:54:54,277] INFO : Plotted jet ratio +rank0 [2026-10-28 16:55:36,066] INFO : Step 46100/50000 rank0 | Train Loss: 7.3680 | Train Loss Unweighted: 2.6031 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3072 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1306 | Regression_eta: 0.0124 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7874 | Classification_binary: 2.3072 | Regression_cos_phi: 0.3754 | Regression_energy: 1.4315 | Regression_eta: 0.6097 | Regression_pt: 1.4982 | Regression_sin_phi: 0.3587 | LR: 1.13e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6795s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0848 | max_backward: 0.2033 | max_optimizer_step: 0.1490 | max_tensorboard_logging: 0.2794 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6469.9 | uss_MB: 5711.7 | child_rss_MB: 3994.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:55:36,280] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:56:17,489] INFO : Step 46200/50000 rank0 | Train Loss: 6.8115 | Train Loss Unweighted: 2.5816 | Train Loss Components: Classification: 0.0155 | Classification_binary: 2.3171 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1155 | Regression_eta: 0.0083 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7183 | Classification_binary: 2.3171 | Regression_cos_phi: 0.3785 | Regression_energy: 1.2654 | Regression_eta: 0.4087 | Regression_pt: 1.3694 | Regression_sin_phi: 0.3541 | LR: 1.13e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6383s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.1532 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6469.9 | uss_MB: 5711.7 | child_rss_MB: 3995.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:56:17,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:56:58,855] INFO : Step 46300/50000 rank0 | Train Loss: 7.1715 | Train Loss Unweighted: 2.5812 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.2874 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1299 | Regression_eta: 0.0080 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7368 | Classification_binary: 2.2874 | Regression_cos_phi: 0.3994 | Regression_energy: 1.4235 | Regression_eta: 0.3949 | Regression_pt: 1.5357 | Regression_sin_phi: 0.3937 | LR: 1.12e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7886s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2619 | max_optimizer_step: 0.1519 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6469.9 | uss_MB: 5711.7 | child_rss_MB: 3998.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:56:59,070] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:57:40,759] INFO : Step 46400/50000 rank0 | Train Loss: 7.1005 | Train Loss Unweighted: 2.5478 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2718 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1210 | Regression_eta: 0.0090 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8092 | Classification_binary: 2.2718 | Regression_cos_phi: 0.4850 | Regression_energy: 1.3257 | Regression_eta: 0.4401 | Regression_pt: 1.3876 | Regression_sin_phi: 0.3812 | LR: 1.11e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.9380s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1043 | max_loss: 0.0035 | max_backward: 0.3946 | max_optimizer_step: 0.1513 | max_tensorboard_logging: 0.3798 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6469.9 | uss_MB: 5713.8 | child_rss_MB: 3997.4 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:57:40,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:58:21,496] INFO : Step 46500/50000 rank0 | Train Loss: 7.8905 | Train Loss Unweighted: 2.7863 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4856 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1294 | Regression_eta: 0.0142 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8659 | Classification_binary: 2.4856 | Regression_cos_phi: 0.4960 | Regression_energy: 1.4182 | Regression_eta: 0.6998 | Regression_pt: 1.4926 | Regression_sin_phi: 0.4323 | LR: 1.11e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8673s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0190 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3095 | max_optimizer_step: 0.1545 | max_tensorboard_logging: 0.4070 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6344.7 | uss_MB: 5585.6 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:58:21,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:02,578] INFO : Step 46600/50000 rank0 | Train Loss: 6.8535 | Train Loss Unweighted: 2.4884 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.2254 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1159 | Regression_eta: 0.0090 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7901 | Classification_binary: 2.2254 | Regression_cos_phi: 0.4145 | Regression_energy: 1.2695 | Regression_eta: 0.4447 | Regression_pt: 1.3053 | Regression_sin_phi: 0.4039 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6817s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1179 | max_loss: 0.0035 | max_backward: 0.2038 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6346.7 | uss_MB: 5587.1 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:59:02,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:44,677] INFO : Step 46700/50000 rank0 | Train Loss: 7.7857 | Train Loss Unweighted: 2.7883 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4895 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1284 | Regression_eta: 0.0120 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8264 | Classification_binary: 2.4895 | Regression_cos_phi: 0.5095 | Regression_energy: 1.4069 | Regression_eta: 0.5892 | Regression_pt: 1.5139 | Regression_sin_phi: 0.4503 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7761s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1829 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2099 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6346.7 | uss_MB: 5587.4 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:59:44,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:00:25,824] INFO : Step 46800/50000 rank0 | Train Loss: 6.9130 | Train Loss Unweighted: 2.5616 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3120 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1045 | Regression_eta: 0.0120 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7752 | Classification_binary: 2.3120 | Regression_cos_phi: 0.4552 | Regression_energy: 1.1455 | Regression_eta: 0.5908 | Regression_pt: 1.2441 | Regression_sin_phi: 0.3901 | LR: 1.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0828 | max_backward: 0.2034 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6346.7 | uss_MB: 5587.6 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:00:26,038] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:06,507] INFO : Step 46900/50000 rank0 | Train Loss: 7.1149 | Train Loss Unweighted: 2.6090 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3426 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1151 | Regression_eta: 0.0111 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8176 | Classification_binary: 2.3426 | Regression_cos_phi: 0.4367 | Regression_energy: 1.2615 | Regression_eta: 0.5434 | Regression_pt: 1.3211 | Regression_sin_phi: 0.3920 | LR: 1.09e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.9438s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0488 | max_forward: 0.1041 | max_loss: 0.0035 | max_backward: 0.3961 | max_optimizer_step: 0.1459 | max_tensorboard_logging: 0.3713 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6346.7 | uss_MB: 5590.1 | child_rss_MB: 3994.1 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 17:01:06,720] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:48,275] INFO : Step 47000/50000 rank0 | Train Loss: 6.9266 | Train Loss Unweighted: 2.6062 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3536 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1071 | Regression_eta: 0.0089 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8743 | Classification_binary: 2.3536 | Regression_cos_phi: 0.4315 | Regression_energy: 1.1739 | Regression_eta: 0.4376 | Regression_pt: 1.2649 | Regression_sin_phi: 0.3909 | LR: 1.08e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7060s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1374 | max_loss: 0.0035 | max_backward: 0.2038 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.2852 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 6346.7 | uss_MB: 5588.1 | child_rss_MB: 3998.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:01:48,490] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:48,509] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 17:01:48,686] INFO : Running validation on rank0 +rank0 [2026-10-28 17:01:48,686] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:01:48,686] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:01:50,013] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 17:01:50,158] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:01:50,291] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:01:50,366] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:01:50,435] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:01:50,504] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:01:50,573] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:01:50,642] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:01:50,755] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:01:50,886] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:01:51,019] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:01:51,090] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:01:51,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:01:51,229] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:01:51,299] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:01:51,370] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:01:51,470] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:01:51,477] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:01:51,479] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:01:51,482] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:01:51,483] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:01:51,486] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:01:51,488] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:01:51,491] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:01:51,492] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:01:51,495] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:01:51,497] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:01:51,500] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:01:51,501] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:01:51,503] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:01:51,504] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:01:51,507] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:01:59,072] INFO : VALIDATION | Step=47000/50000 | Train Loss=6.9266 | Valid Loss=7.3067 | Train Loss Unweighted=2.6062 | Valid Loss Unweighted=2.6053 | Stale=0 | ETA=22.3m +rank0 [2026-10-28 17:01:59,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:01:59,138] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:01:59,142] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:01:59,325] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:02:01,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:02:02,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:02:02,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:02:03,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:02:03,623] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:02:03,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:02:03,697] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:02:03,700] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:02:03,809] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:02:05,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:02:06,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:02:07,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:02:07,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:02:07,987] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:02:07,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:02:08,065] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:02:08,068] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:02:08,178] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:02:10,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:02:11,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:02:11,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:02:12,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:02:12,590] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:02:12,598] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:02:12,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:02:13,269] INFO : Plotted number of elements +rank0 [2026-10-28 17:02:14,719] INFO : Plotted jets +rank0 [2026-10-28 17:02:15,821] INFO : Plotted jet ratio +rank0 [2026-10-28 17:02:15,854] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:02:16,082] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:02:16,475] INFO : Plotted number of elements +rank0 [2026-10-28 17:02:17,864] INFO : Plotted jets +rank0 [2026-10-28 17:02:18,990] INFO : Plotted jet ratio +rank0 [2026-10-28 17:02:19,024] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 17:02:19,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 17:02:20,243] INFO : Plotted number of elements +rank0 [2026-10-28 17:02:21,623] INFO : Plotted jets +rank0 [2026-10-28 17:02:22,695] INFO : Plotted jet ratio +rank0 [2026-10-28 17:03:04,724] INFO : Step 47100/50000 rank0 | Train Loss: 7.3951 | Train Loss Unweighted: 2.6919 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.4087 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1224 | Regression_eta: 0.0100 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7692 | Classification_binary: 2.4087 | Regression_cos_phi: 0.5522 | Regression_energy: 1.3409 | Regression_eta: 0.4904 | Regression_pt: 1.4415 | Regression_sin_phi: 0.3922 | LR: 1.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1137 | max_loss: 0.0038 | max_backward: 0.2039 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:03:04,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:03:46,160] INFO : Step 47200/50000 rank0 | Train Loss: 7.3369 | Train Loss Unweighted: 2.5812 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.2925 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1256 | Regression_eta: 0.0125 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7889 | Classification_binary: 2.2925 | Regression_cos_phi: 0.3871 | Regression_energy: 1.3762 | Regression_eta: 0.6162 | Regression_pt: 1.4570 | Regression_sin_phi: 0.4190 | LR: 1.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1530 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.2973 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3995.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:03:46,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:04:28,162] INFO : Step 47300/50000 rank0 | Train Loss: 8.0476 | Train Loss Unweighted: 2.6238 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3205 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1271 | Regression_eta: 0.0201 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8781 | Classification_binary: 2.3205 | Regression_cos_phi: 0.5362 | Regression_energy: 1.3927 | Regression_eta: 0.9862 | Regression_pt: 1.4651 | Regression_sin_phi: 0.4688 | LR: 1.06e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7757s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1850 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.2794 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3995.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:04:28,377] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:05:09,873] INFO : Step 47400/50000 rank0 | Train Loss: 6.8776 | Train Loss Unweighted: 2.4663 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.1906 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1209 | Regression_eta: 0.0080 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7885 | Classification_binary: 2.1906 | Regression_cos_phi: 0.3671 | Regression_energy: 1.3245 | Regression_eta: 0.3957 | Regression_pt: 1.4185 | Regression_sin_phi: 0.3926 | LR: 1.06e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.9385s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1216 | max_loss: 0.0039 | max_backward: 0.3930 | max_optimizer_step: 0.1500 | max_tensorboard_logging: 0.3746 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6556.0 | uss_MB: 5798.5 | child_rss_MB: 3994.9 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 17:05:10,085] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:05:51,662] INFO : Step 47500/50000 rank0 | Train Loss: 7.5574 | Train Loss Unweighted: 2.6618 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.3583 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1329 | Regression_eta: 0.0090 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7586 | Classification_binary: 2.3583 | Regression_cos_phi: 0.4805 | Regression_energy: 1.4561 | Regression_eta: 0.4407 | Regression_pt: 1.5665 | Regression_sin_phi: 0.4968 | LR: 1.06e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1809 | max_loss: 0.0037 | max_backward: 0.2036 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:05:51,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:33,719] INFO : Step 47600/50000 rank0 | Train Loss: 7.2782 | Train Loss Unweighted: 2.5998 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3153 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1228 | Regression_eta: 0.0076 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8113 | Classification_binary: 2.3153 | Regression_cos_phi: 0.5821 | Regression_energy: 1.3456 | Regression_eta: 0.3759 | Regression_pt: 1.4660 | Regression_sin_phi: 0.3820 | LR: 1.05e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6573s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0796 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6419.2 | uss_MB: 5659.2 | child_rss_MB: 3996.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:06:33,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:07:14,841] INFO : Step 47700/50000 rank0 | Train Loss: 7.4307 | Train Loss Unweighted: 2.5335 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2494 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1236 | Regression_eta: 0.0165 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7574 | Classification_binary: 2.2494 | Regression_cos_phi: 0.4882 | Regression_energy: 1.3543 | Regression_eta: 0.8115 | Regression_pt: 1.3714 | Regression_sin_phi: 0.3986 | LR: 1.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7976s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.2740 | max_optimizer_step: 0.1516 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6419.2 | uss_MB: 5659.3 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:07:15,057] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:07:56,649] INFO : Step 47800/50000 rank0 | Train Loss: 7.1418 | Train Loss Unweighted: 2.6018 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3357 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1153 | Regression_eta: 0.0118 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8480 | Classification_binary: 2.3357 | Regression_cos_phi: 0.4073 | Regression_energy: 1.2631 | Regression_eta: 0.5782 | Regression_pt: 1.3025 | Regression_sin_phi: 0.4070 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2788s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.6627 | max_loss: 0.0037 | max_backward: 0.2038 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.2847 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6420.2 | uss_MB: 5659.9 | child_rss_MB: 3997.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:07:56,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:08:37,742] INFO : Step 47900/50000 rank0 | Train Loss: 7.1295 | Train Loss Unweighted: 2.4979 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.2232 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1181 | Regression_eta: 0.0124 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7613 | Classification_binary: 2.2232 | Regression_cos_phi: 0.4629 | Regression_energy: 1.2942 | Regression_eta: 0.6102 | Regression_pt: 1.3774 | Regression_sin_phi: 0.4002 | LR: 1.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7195s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.2196 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6420.2 | uss_MB: 5659.9 | child_rss_MB: 3992.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 17:08:37,957] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:09:17,752] INFO : Step 48000/50000 rank0 | Train Loss: 6.8565 | Train Loss Unweighted: 2.5352 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2814 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1084 | Regression_eta: 0.0097 | Regression_pt: 0.1059 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8103 | Classification_binary: 2.2814 | Regression_cos_phi: 0.4439 | Regression_energy: 1.1872 | Regression_eta: 0.4787 | Regression_pt: 1.2697 | Regression_sin_phi: 0.3853 | LR: 1.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6191s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0153 | max_backward: 0.2034 | max_optimizer_step: 0.1517 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6420.2 | uss_MB: 5660.1 | child_rss_MB: 3992.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:09:17,966] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:09:17,985] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 17:09:18,166] INFO : Running validation on rank0 +rank0 [2026-10-28 17:09:18,167] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:09:18,167] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:09:19,557] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 17:09:19,706] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:09:19,837] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:09:19,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:09:19,981] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:09:20,050] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:09:20,120] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:09:20,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:09:20,303] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:09:20,435] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:09:20,567] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:09:20,638] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:09:20,708] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:09:20,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:09:20,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:09:20,916] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:09:21,015] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:09:21,022] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:09:21,024] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:09:21,027] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:09:21,028] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:09:21,031] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:09:21,032] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:09:21,035] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:09:21,037] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:09:21,039] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:09:21,041] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:09:21,042] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:09:21,045] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:09:21,048] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:09:21,050] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:09:21,052] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:09:28,763] INFO : VALIDATION | Step=48000/50000 | Train Loss=6.8565 | Valid Loss=7.2882 | Train Loss Unweighted=2.5352 | Valid Loss Unweighted=2.6060 | Stale=0 | ETA=14.9m +rank0 [2026-10-28 17:09:28,774] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:28,920] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:09:28,924] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:09:29,116] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:09:31,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:09:31,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:09:32,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:09:33,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:09:33,577] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:09:33,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:33,655] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:09:33,658] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:09:33,767] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:09:36,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:09:36,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:09:37,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:09:38,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:09:38,254] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:09:38,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:38,327] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:09:38,330] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:09:38,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:09:40,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:09:41,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:09:41,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:09:42,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:09:42,529] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:09:42,537] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:09:42,798] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:09:43,214] INFO : Plotted number of elements +rank0 [2026-10-28 17:09:44,659] INFO : Plotted jets +rank0 [2026-10-28 17:09:45,764] INFO : Plotted jet ratio +rank0 [2026-10-28 17:09:45,797] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:09:46,031] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:09:46,424] INFO : Plotted number of elements +rank0 [2026-10-28 17:09:47,824] INFO : Plotted jets +rank0 [2026-10-28 17:09:49,563] INFO : Plotted jet ratio +rank0 [2026-10-28 17:09:49,596] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 17:09:49,807] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 17:09:50,171] INFO : Plotted number of elements +rank0 [2026-10-28 17:09:51,556] INFO : Plotted jets +rank0 [2026-10-28 17:09:52,632] INFO : Plotted jet ratio +rank0 [2026-10-28 17:10:34,996] INFO : Step 48100/50000 rank0 | Train Loss: 7.2827 | Train Loss Unweighted: 2.6399 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3723 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1147 | Regression_eta: 0.0127 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7825 | Classification_binary: 2.3723 | Regression_cos_phi: 0.4974 | Regression_energy: 1.2568 | Regression_eta: 0.6220 | Regression_pt: 1.3108 | Regression_sin_phi: 0.4409 | LR: 1.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.0407s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1024 | max_forward: 0.1044 | max_loss: 0.0035 | max_backward: 0.3940 | max_optimizer_step: 0.1521 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6604.5 | uss_MB: 5847.7 | child_rss_MB: 3989.5 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 17:10:35,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:11:15,904] INFO : Step 48200/50000 rank0 | Train Loss: 7.2577 | Train Loss Unweighted: 2.5515 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2676 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1246 | Regression_eta: 0.0102 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7932 | Classification_binary: 2.2676 | Regression_cos_phi: 0.4694 | Regression_energy: 1.3656 | Regression_eta: 0.4992 | Regression_pt: 1.4198 | Regression_sin_phi: 0.4429 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6713s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3995.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:11:16,118] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:11:57,564] INFO : Step 48300/50000 rank0 | Train Loss: 7.2231 | Train Loss Unweighted: 2.5918 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3117 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1211 | Regression_eta: 0.0090 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8550 | Classification_binary: 2.3117 | Regression_cos_phi: 0.4174 | Regression_energy: 1.3272 | Regression_eta: 0.4439 | Regression_pt: 1.4207 | Regression_sin_phi: 0.4472 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8188s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0780 | max_loss: 0.0037 | max_backward: 0.3066 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 0.3720 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 17:11:57,781] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:12:39,251] INFO : Step 48400/50000 rank0 | Train Loss: 7.8885 | Train Loss Unweighted: 2.6583 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3429 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1387 | Regression_eta: 0.0096 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9110 | Classification_binary: 2.3429 | Regression_cos_phi: 0.6339 | Regression_energy: 1.5200 | Regression_eta: 0.4730 | Regression_pt: 1.5792 | Regression_sin_phi: 0.4284 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7923s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.2030 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 0.2855 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3996.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:12:39,464] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:13:20,360] INFO : Step 48500/50000 rank0 | Train Loss: 6.9111 | Train Loss Unweighted: 2.5033 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2449 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1121 | Regression_eta: 0.0123 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7725 | Classification_binary: 2.2449 | Regression_cos_phi: 0.4318 | Regression_energy: 1.2278 | Regression_eta: 0.6040 | Regression_pt: 1.2658 | Regression_sin_phi: 0.3643 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7421s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0779 | max_loss: 0.0037 | max_backward: 0.2029 | max_optimizer_step: 0.1539 | max_tensorboard_logging: 0.3762 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3994.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:13:20,574] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:01,026] INFO : Step 48600/50000 rank0 | Train Loss: 6.6640 | Train Loss Unweighted: 2.4674 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.2130 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1106 | Regression_eta: 0.0074 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8750 | Classification_binary: 2.2130 | Regression_cos_phi: 0.3803 | Regression_energy: 1.2120 | Regression_eta: 0.3630 | Regression_pt: 1.2793 | Regression_sin_phi: 0.3414 | LR: 1.02e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6422s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0377 | max_backward: 0.2031 | max_optimizer_step: 0.1500 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6493.2 | uss_MB: 5734.8 | child_rss_MB: 3994.0 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:14:01,239] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:41,982] INFO : Step 48700/50000 rank0 | Train Loss: 7.1915 | Train Loss Unweighted: 2.6255 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3568 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1165 | Regression_eta: 0.0101 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8297 | Classification_binary: 2.3568 | Regression_cos_phi: 0.4324 | Regression_energy: 1.2762 | Regression_eta: 0.4954 | Regression_pt: 1.3273 | Regression_sin_phi: 0.4738 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8166s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1849 | max_loss: 0.0037 | max_backward: 0.2033 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6493.2 | uss_MB: 5735.1 | child_rss_MB: 3991.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:14:42,195] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:15:23,121] INFO : Step 48800/50000 rank0 | Train Loss: 6.7459 | Train Loss Unweighted: 2.4737 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.2159 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1113 | Regression_eta: 0.0071 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7456 | Classification_binary: 2.2159 | Regression_cos_phi: 0.4682 | Regression_energy: 1.2199 | Regression_eta: 0.3477 | Regression_pt: 1.3176 | Regression_sin_phi: 0.4311 | LR: 1.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8162s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1043 | max_loss: 0.1048 | max_backward: 0.2876 | max_optimizer_step: 0.1479 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6493.2 | uss_MB: 5737.5 | child_rss_MB: 3996.3 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 17:15:23,335] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:16:04,511] INFO : Step 48900/50000 rank0 | Train Loss: 7.3672 | Train Loss Unweighted: 2.6951 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4193 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1214 | Regression_eta: 0.0110 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8226 | Classification_binary: 2.4193 | Regression_cos_phi: 0.5183 | Regression_energy: 1.3307 | Regression_eta: 0.5413 | Regression_pt: 1.3436 | Regression_sin_phi: 0.3914 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8409s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0284 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3090 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 0.3764 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6493.2 | uss_MB: 5735.5 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:16:04,723] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:16:45,370] INFO : Step 49000/50000 rank0 | Train Loss: 6.8608 | Train Loss Unweighted: 2.5118 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2559 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1103 | Regression_eta: 0.0096 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7934 | Classification_binary: 2.2559 | Regression_cos_phi: 0.4729 | Regression_energy: 1.2086 | Regression_eta: 0.4737 | Regression_pt: 1.2720 | Regression_sin_phi: 0.3844 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8192s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0037 | max_backward: 0.3049 | max_optimizer_step: 0.1524 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6493.2 | uss_MB: 5735.6 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 17:16:45,585] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:16:45,616] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 17:16:45,796] INFO : Running validation on rank0 +rank0 [2026-10-28 17:16:45,797] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:16:45,797] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:16:47,132] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 17:16:47,280] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:16:47,412] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:16:47,487] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:16:47,557] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:16:47,627] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:16:47,697] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:16:47,767] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:16:47,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:16:48,014] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:16:48,147] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:16:48,219] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:16:48,289] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:16:48,358] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:16:48,427] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:16:48,497] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:16:48,598] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:16:48,605] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:16:48,607] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:16:48,610] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:16:48,611] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:16:48,614] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:16:48,617] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:16:48,618] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:16:48,620] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:16:48,623] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:16:48,624] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:16:48,627] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:16:48,630] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:16:48,631] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:16:48,633] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:16:48,636] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:16:56,258] INFO : VALIDATION | Step=49000/50000 | Train Loss=6.8608 | Valid Loss=7.2898 | Train Loss Unweighted=2.5118 | Valid Loss Unweighted=2.6059 | Stale=1 | ETA=7.4m +rank0 [2026-10-28 17:16:56,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:16:56,368] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:16:56,371] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:16:56,623] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:16:58,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:16:59,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:16:59,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:17:00,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:17:00,751] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:17:00,754] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:17:00,808] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:17:00,811] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:17:00,938] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:17:03,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:17:03,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:17:04,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:17:05,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:17:05,350] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:17:05,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:17:05,407] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:17:05,410] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:17:05,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:17:07,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:17:08,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:17:08,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:17:09,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:17:09,584] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:17:09,592] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:17:09,856] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:17:10,267] INFO : Plotted number of elements +rank0 [2026-10-28 17:17:11,714] INFO : Plotted jets +rank0 [2026-10-28 17:17:12,822] INFO : Plotted jet ratio +rank0 [2026-10-28 17:17:12,856] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:17:13,085] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:17:13,480] INFO : Plotted number of elements +rank0 [2026-10-28 17:17:14,876] INFO : Plotted jets +rank0 [2026-10-28 17:17:16,653] INFO : Plotted jet ratio +rank0 [2026-10-28 17:17:16,686] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 17:17:16,892] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 17:17:17,255] INFO : Plotted number of elements +rank0 [2026-10-28 17:17:18,630] INFO : Plotted jets +rank0 [2026-10-28 17:17:19,695] INFO : Plotted jet ratio +rank0 [2026-10-28 17:18:00,771] INFO : Step 49100/50000 rank0 | Train Loss: 7.1387 | Train Loss Unweighted: 2.6347 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.3740 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1119 | Regression_eta: 0.0124 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7431 | Classification_binary: 2.3740 | Regression_cos_phi: 0.4947 | Regression_energy: 1.2264 | Regression_eta: 0.6103 | Regression_pt: 1.2807 | Regression_sin_phi: 0.4094 | LR: 1.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6690s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0167 | max_backward: 0.2038 | max_optimizer_step: 0.1541 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6685.8 | uss_MB: 5927.5 | child_rss_MB: 3990.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:18:00,982] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:18:42,758] INFO : Step 49200/50000 rank0 | Train Loss: 7.0735 | Train Loss Unweighted: 2.5507 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.2835 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1174 | Regression_eta: 0.0112 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7065 | Classification_binary: 2.2835 | Regression_cos_phi: 0.5262 | Regression_energy: 1.2859 | Regression_eta: 0.5528 | Regression_pt: 1.3118 | Regression_sin_phi: 0.4067 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8464s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0785 | max_loss: 0.0036 | max_backward: 0.3066 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.3809 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6685.8 | uss_MB: 5927.6 | child_rss_MB: 3991.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:18:42,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:19:23,632] INFO : Step 49300/50000 rank0 | Train Loss: 6.9003 | Train Loss Unweighted: 2.4761 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.2115 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1142 | Regression_eta: 0.0107 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7326 | Classification_binary: 2.2115 | Regression_cos_phi: 0.5070 | Regression_energy: 1.2514 | Regression_eta: 0.5256 | Regression_pt: 1.3346 | Regression_sin_phi: 0.3375 | LR: 1.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7752s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1830 | max_loss: 0.0037 | max_backward: 0.2031 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6685.8 | uss_MB: 5927.6 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:19:23,843] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:20:05,391] INFO : Step 49400/50000 rank0 | Train Loss: 7.6710 | Train Loss Unweighted: 2.7216 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4271 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1276 | Regression_eta: 0.0108 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9005 | Classification_binary: 2.4271 | Regression_cos_phi: 0.5206 | Regression_energy: 1.3979 | Regression_eta: 0.5302 | Regression_pt: 1.4698 | Regression_sin_phi: 0.4248 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2704 | max_optimizer_step: 0.1544 | max_tensorboard_logging: 0.3812 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6685.8 | uss_MB: 5927.6 | child_rss_MB: 3994.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:20:05,608] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:20:45,907] INFO : Step 49500/50000 rank0 | Train Loss: 6.8956 | Train Loss Unweighted: 2.6684 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.4125 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1103 | Regression_eta: 0.0081 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7851 | Classification_binary: 2.4125 | Regression_cos_phi: 0.4595 | Regression_energy: 1.2085 | Regression_eta: 0.3975 | Regression_pt: 1.3059 | Regression_sin_phi: 0.3267 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.0435s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1523 | max_loss: 0.0036 | max_backward: 0.3909 | max_optimizer_step: 0.1568 | max_tensorboard_logging: 0.3955 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6685.8 | uss_MB: 5929.7 | child_rss_MB: 3992.1 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 17:20:46,120] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:21:27,902] INFO : Step 49600/50000 rank0 | Train Loss: 7.4177 | Train Loss Unweighted: 2.6458 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3490 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1327 | Regression_eta: 0.0095 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8389 | Classification_binary: 2.3490 | Regression_cos_phi: 0.4292 | Regression_energy: 1.4537 | Regression_eta: 0.4690 | Regression_pt: 1.4897 | Regression_sin_phi: 0.3881 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8542s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3107 | max_optimizer_step: 0.1523 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6565.4 | uss_MB: 5807.4 | child_rss_MB: 3992.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 17:21:28,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:22:08,664] INFO : Step 49700/50000 rank0 | Train Loss: 7.0558 | Train Loss Unweighted: 2.6374 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3657 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1184 | Regression_eta: 0.0089 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8113 | Classification_binary: 2.3657 | Regression_cos_phi: 0.3240 | Regression_energy: 1.2973 | Regression_eta: 0.4375 | Regression_pt: 1.3839 | Regression_sin_phi: 0.4361 | LR: 1.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7747s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2478 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.3825 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6566.4 | uss_MB: 5807.9 | child_rss_MB: 3990.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:22:08,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:22:50,571] INFO : Step 49800/50000 rank0 | Train Loss: 7.5939 | Train Loss Unweighted: 2.6136 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3335 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1196 | Regression_eta: 0.0219 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7707 | Classification_binary: 2.3335 | Regression_cos_phi: 0.4741 | Regression_energy: 1.3103 | Regression_eta: 1.0781 | Regression_pt: 1.3247 | Regression_sin_phi: 0.3026 | LR: 1.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8773s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0274 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.3093 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.3987 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6566.4 | uss_MB: 5808.0 | child_rss_MB: 3991.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:22:50,786] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:23:32,462] INFO : Step 49900/50000 rank0 | Train Loss: 7.3435 | Train Loss Unweighted: 2.5946 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2906 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1331 | Regression_eta: 0.0117 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7555 | Classification_binary: 2.2906 | Regression_cos_phi: 0.3533 | Regression_energy: 1.4589 | Regression_eta: 0.5766 | Regression_pt: 1.5923 | Regression_sin_phi: 0.3165 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1046 | max_loss: 0.0036 | max_backward: 0.2871 | max_optimizer_step: 0.1502 | max_tensorboard_logging: 0.3011 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6566.4 | uss_MB: 5810.3 | child_rss_MB: 3991.9 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 96.9 +rank0 [2026-10-28 17:23:32,675] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:24:14,060] INFO : Step 50000/50000 rank0 | Train Loss: 7.2474 | Train Loss Unweighted: 2.5476 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.2704 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1197 | Regression_eta: 0.0126 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7453 | Classification_binary: 2.2704 | Regression_cos_phi: 0.4632 | Regression_energy: 1.3118 | Regression_eta: 0.6171 | Regression_pt: 1.3791 | Regression_sin_phi: 0.4605 | LR: 1.00e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8056s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1823 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6566.4 | uss_MB: 5808.3 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:24:14,274] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:24:14,295] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 17:24:14,474] INFO : Running validation on rank0 +rank0 [2026-10-28 17:24:14,475] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:24:14,475] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:24:15,808] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 17:24:15,956] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:24:16,089] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:24:16,164] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:24:16,233] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:24:16,302] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:24:16,372] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:24:16,442] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:24:16,555] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:24:16,687] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:24:16,820] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:24:16,892] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:24:16,964] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:24:17,032] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:24:17,103] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:24:17,173] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:24:17,273] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:24:17,279] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:24:17,282] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:24:17,285] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:24:17,286] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:24:17,288] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:24:17,289] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:24:17,292] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:24:17,295] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:24:17,298] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:24:17,299] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:24:17,301] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:24:17,304] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:24:17,305] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:24:17,307] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:24:17,310] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:24:25,080] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.2474 | Valid Loss=7.2727 | Train Loss Unweighted=2.5476 | Valid Loss Unweighted=2.6010 | Stale=0 | ETA=0.0m +rank0 [2026-10-28 17:24:25,090] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:24:25,148] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:24:25,151] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:24:25,338] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:24:27,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:24:28,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:24:28,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:24:29,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:24:29,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:24:29,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:24:29,675] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:24:29,678] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:24:29,808] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:24:31,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:24:32,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:24:33,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:24:34,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:24:34,277] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:24:34,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:24:34,334] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:24:34,337] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:24:34,464] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:24:36,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:24:37,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:24:37,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:24:38,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:24:38,533] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:24:38,541] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:24:38,799] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:24:39,209] INFO : Plotted number of elements +rank0 [2026-10-28 17:24:40,644] INFO : Plotted jets +rank0 [2026-10-28 17:24:41,742] INFO : Plotted jet ratio +rank0 [2026-10-28 17:24:41,774] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:24:42,005] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:24:42,400] INFO : Plotted number of elements +rank0 [2026-10-28 17:24:43,787] INFO : Plotted jets +rank0 [2026-10-28 17:24:45,530] INFO : Plotted jet ratio +rank0 [2026-10-28 17:24:45,579] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 17:24:45,790] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 17:24:46,154] INFO : Plotted number of elements +rank0 [2026-10-28 17:24:47,535] INFO : Plotted jets +rank0 [2026-10-28 17:24:48,604] INFO : Plotted jet ratio +rank0 [2026-10-28 17:24:49,126] INFO : Training completed. Total time on device 0: 372.263min diff --git a/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train.log.0 b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..1309f9cff5bfcad1dba319fbe32bac412d9d3d3d --- /dev/null +++ b/cld/clusters/v3.2.0/cld_pf_heptv2_seed12345_20261028_111202_024020/train.log.0 @@ -0,0 +1,13929 @@ +rank0 [2026-10-28 11:12:24,881] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,882] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,882] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,882] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,882] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,918] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,919] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,971] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,979] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,986] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,993] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:25,001] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:25,008] INFO : Backbone initialization took 0.09s +rank0 [2026-10-28 11:12:25,008] INFO : backbone parameters: 7901376 +rank0 [2026-10-28 11:12:25,009] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:25,013] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:25,013] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:25,013] INFO : jagged_attention enabled=False +rank0 [2026-10-28 11:12:25,013] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-28 11:12:25,014] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:25,014] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:25,014] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:25,014] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:25,014] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:25,014] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:25,014] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:25,015] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,015] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,015] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:25,015] INFO : Total MLPF parameters: 8639954 +rank0 [2026-10-28 11:12:25,015] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:25,015] INFO : Instantiated model +rank0 [2026-10-28 11:12:25,016] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,241] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,243] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,355] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +205 nn_energy.nn.1.0.bias 256 0 +206 nn_energy.nn.1.2.weight 256 0 +207 nn_energy.nn.1.2.bias 256 0 +208 nn_energy.nn.1.4.weight 256 0 +209 nn_energy.nn.1.4.bias 1 0 + +[210 rows x 3 columns] +rank0 [2026-10-28 11:12:25,356] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,357] INFO : Trainable parameters: 8639954 +rank0 [2026-10-28 11:12:25,357] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-28 11:12:25,357] INFO : Total parameters: 8663222 +rank0 [2026-10-28 11:12:25,360] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.regions 0 96 + backbone.0.attn.out_linear.weight 65536 0 + backbone.0.attn.out_linear.bias 256 0 + backbone.0.attn.e2lsh.alpha 0 864 + backbone.0.attn.e2lsh.beta 0 3 + backbone.0.attn.e2lsh_new.alpha 0 96 + backbone.0.attn.e2lsh_new.beta 0 3 + backbone.0.w_q.weight 65536 0 + backbone.0.w_k.weight 65536 0 + backbone.0.w_v.weight 65536 0 + backbone.0.w_rpe.weight 0 2560 + backbone.0.w_rpe.bias 0 256 +backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + backbone.0.pe_func.position_embedding_head.0.bias 1024 0 +backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + backbone.0.pe_func.position_embedding_head.1.bias 1024 0 +backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + backbone.0.pe_func.position_embedding_head.3.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm2.weight 256 0 + backbone.0.q_norm.weight 16 0 + backbone.0.k_norm.weight 16 0 + backbone.0.ff.gate_proj.weight 262144 0 + backbone.0.ff.up_proj.weight 262144 0 + backbone.0.ff.down_proj.weight 262144 0 + backbone.1.regions 0 96 + backbone.1.attn.out_linear.weight 65536 0 + backbone.1.attn.out_linear.bias 256 0 + backbone.1.attn.e2lsh.alpha 0 864 + backbone.1.attn.e2lsh.beta 0 3 + backbone.1.attn.e2lsh_new.alpha 0 96 + backbone.1.attn.e2lsh_new.beta 0 3 + backbone.1.w_q.weight 65536 0 + backbone.1.w_k.weight 65536 0 + backbone.1.w_v.weight 65536 0 + backbone.1.w_rpe.weight 0 2560 + backbone.1.w_rpe.bias 0 256 +backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + backbone.1.pe_func.position_embedding_head.0.bias 1024 0 +backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + backbone.1.pe_func.position_embedding_head.1.bias 1024 0 +backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + backbone.1.pe_func.position_embedding_head.3.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm2.weight 256 0 + backbone.1.q_norm.weight 16 0 + backbone.1.k_norm.weight 16 0 + backbone.1.ff.gate_proj.weight 262144 0 + backbone.1.ff.up_proj.weight 262144 0 + backbone.1.ff.down_proj.weight 262144 0 + backbone.2.regions 0 96 + backbone.2.attn.out_linear.weight 65536 0 + backbone.2.attn.out_linear.bias 256 0 + backbone.2.attn.e2lsh.alpha 0 864 + backbone.2.attn.e2lsh.beta 0 3 + backbone.2.attn.e2lsh_new.alpha 0 96 + backbone.2.attn.e2lsh_new.beta 0 3 + backbone.2.w_q.weight 65536 0 + backbone.2.w_k.weight 65536 0 + backbone.2.w_v.weight 65536 0 + backbone.2.w_rpe.weight 0 2560 + backbone.2.w_rpe.bias 0 256 +backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + backbone.2.pe_func.position_embedding_head.0.bias 1024 0 +backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + backbone.2.pe_func.position_embedding_head.1.bias 1024 0 +backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + backbone.2.pe_func.position_embedding_head.3.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm2.weight 256 0 + backbone.2.q_norm.weight 16 0 + backbone.2.k_norm.weight 16 0 + backbone.2.ff.gate_proj.weight 262144 0 + backbone.2.ff.up_proj.weight 262144 0 + backbone.2.ff.down_proj.weight 262144 0 + backbone.3.regions 0 96 + backbone.3.attn.out_linear.weight 65536 0 + backbone.3.attn.out_linear.bias 256 0 + backbone.3.attn.e2lsh.alpha 0 864 + backbone.3.attn.e2lsh.beta 0 3 + backbone.3.attn.e2lsh_new.alpha 0 96 + backbone.3.attn.e2lsh_new.beta 0 3 + backbone.3.w_q.weight 65536 0 + backbone.3.w_k.weight 65536 0 + backbone.3.w_v.weight 65536 0 + backbone.3.w_rpe.weight 0 2560 + backbone.3.w_rpe.bias 0 256 +backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + backbone.3.pe_func.position_embedding_head.0.bias 1024 0 +backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + backbone.3.pe_func.position_embedding_head.1.bias 1024 0 +backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + backbone.3.pe_func.position_embedding_head.3.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm2.weight 256 0 + backbone.3.q_norm.weight 16 0 + backbone.3.k_norm.weight 16 0 + backbone.3.ff.gate_proj.weight 262144 0 + backbone.3.ff.up_proj.weight 262144 0 + backbone.3.ff.down_proj.weight 262144 0 + backbone.4.regions 0 96 + backbone.4.attn.out_linear.weight 65536 0 + backbone.4.attn.out_linear.bias 256 0 + backbone.4.attn.e2lsh.alpha 0 864 + backbone.4.attn.e2lsh.beta 0 3 + backbone.4.attn.e2lsh_new.alpha 0 96 + backbone.4.attn.e2lsh_new.beta 0 3 + backbone.4.w_q.weight 65536 0 + backbone.4.w_k.weight 65536 0 + backbone.4.w_v.weight 65536 0 + backbone.4.w_rpe.weight 0 2560 + backbone.4.w_rpe.bias 0 256 +backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + backbone.4.pe_func.position_embedding_head.0.bias 1024 0 +backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + backbone.4.pe_func.position_embedding_head.1.bias 1024 0 +backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + backbone.4.pe_func.position_embedding_head.3.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm2.weight 256 0 + backbone.4.q_norm.weight 16 0 + backbone.4.k_norm.weight 16 0 + backbone.4.ff.gate_proj.weight 262144 0 + backbone.4.ff.up_proj.weight 262144 0 + backbone.4.ff.down_proj.weight 262144 0 + backbone.5.regions 0 96 + backbone.5.attn.out_linear.weight 65536 0 + backbone.5.attn.out_linear.bias 256 0 + backbone.5.attn.e2lsh.alpha 0 864 + backbone.5.attn.e2lsh.beta 0 3 + backbone.5.attn.e2lsh_new.alpha 0 96 + backbone.5.attn.e2lsh_new.beta 0 3 + backbone.5.w_q.weight 65536 0 + backbone.5.w_k.weight 65536 0 + backbone.5.w_v.weight 65536 0 + backbone.5.w_rpe.weight 0 2560 + backbone.5.w_rpe.bias 0 256 +backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + backbone.5.pe_func.position_embedding_head.0.bias 1024 0 +backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + backbone.5.pe_func.position_embedding_head.1.bias 1024 0 +backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + backbone.5.pe_func.position_embedding_head.3.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm2.weight 256 0 + backbone.5.q_norm.weight 16 0 + backbone.5.k_norm.weight 16 0 + backbone.5.ff.gate_proj.weight 262144 0 + backbone.5.ff.up_proj.weight 262144 0 + backbone.5.ff.down_proj.weight 262144 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,364] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020 +rank0 [2026-10-28 11:12:25,364] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020 +rank0 [2026-10-28 11:12:25,364] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,364] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,276] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,312] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,343] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,375] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,402] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,442] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,464] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,490] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,514] INFO : train_dataset: cld_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,542] INFO : train_dataset: cld_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,542] INFO : sample=cld_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,561] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,596] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,646] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,694] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,733] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,762] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,789] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,815] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,840] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,865] INFO : train_dataset: cld_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,865] INFO : sample=cld_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,901] INFO : train_dataset: cld_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:32,923] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,966] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:32,993] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,016] INFO : train_dataset: cld_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,052] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,080] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,122] INFO : train_dataset: cld_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,148] INFO : train_dataset: cld_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset: cld_edm_qq_pf, 90598 +rank0 [2026-10-28 11:12:33,177] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,178] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,178] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,178] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,183] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,189] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,195] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,200] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,206] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,211] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,217] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,223] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,228] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,234] INFO : valid_dataset: cld_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,234] INFO : sample=cld_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,239] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,244] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,249] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,254] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,259] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,264] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,269] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,274] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,279] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,284] INFO : valid_dataset: cld_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,285] INFO : sample=cld_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,289] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,294] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,299] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,304] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,309] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,314] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,319] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,324] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,329] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,334] INFO : valid_dataset: cld_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,334] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,334] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,361] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,365] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,365] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-28 11:12:33,365] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,365] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,371] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,389] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,641] INFO : Created train_iterator +rank0 [2026-10-28 11:13:44,705] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 5.158649005889893, 'Classification': 0.11118821758776903, 'Regression_pt': 0.43017278850078583, 'Regression_eta': 0.10493254894390702, 'Regression_sin_phi': 0.08165183812379836, 'Regression_cos_phi': 0.07327452296391129, 'Regression_energy': 0.4708075523376465}, weights={'Classification_binary': 1.0, 'Classification': 46.39564514160156, 'Regression_pt': 11.992039680480957, 'Regression_eta': 49.16157150268555, 'Regression_sin_phi': 63.17860412597656, 'Regression_cos_phi': 70.40167236328125, 'Regression_energy': 10.957022666931152} +rank0 [2026-10-28 11:13:45,891] INFO : Step 100/50000 rank0 | Train Loss: 5.4789 | Train Loss Unweighted: 5.4789 | Train Loss Components: Classification: 0.0486 | Classification_binary: 4.6932 | Regression_cos_phi: 0.0244 | Regression_energy: 0.3346 | Regression_eta: 0.0324 | Regression_pt: 0.3264 | Regression_sin_phi: 0.0193 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 4.6932 | Regression_cos_phi: 0.0244 | Regression_energy: 0.3346 | Regression_eta: 0.0324 | Regression_pt: 0.3264 | Regression_sin_phi: 0.0193 | LR: 1.00e-03 | DataLoad Time: 0.0018s | Model Forward Time: 1.6840s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.1291 | max_backward: 0.2024 | max_optimizer_step: 0.1173 | max_tensorboard_logging: 0.4304 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 2948.8 | uss_MB: 2142.2 | child_rss_MB: 18193.7 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:13:46,104] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:14:26,674] INFO : Step 200/50000 rank0 | Train Loss: 16.9002 | Train Loss Unweighted: 5.1575 | Train Loss Components: Classification: 0.0411 | Classification_binary: 4.4483 | Regression_cos_phi: 0.0126 | Regression_energy: 0.3124 | Regression_eta: 0.0350 | Regression_pt: 0.2921 | Regression_sin_phi: 0.0161 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.9087 | Classification_binary: 4.4483 | Regression_cos_phi: 0.8847 | Regression_energy: 3.4233 | Regression_eta: 1.7183 | Regression_pt: 3.5028 | Regression_sin_phi: 1.0142 | LR: 1.00e-03 | DataLoad Time: 0.0025s | Model Forward Time: 1.3585s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.3098 | max_optimizer_step: 0.1065 | max_tensorboard_logging: 0.2600 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 2948.5 | uss_MB: 2142.7 | child_rss_MB: 4127.1 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:14:26,888] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:07,444] INFO : Step 300/50000 rank0 | Train Loss: 15.8682 | Train Loss Unweighted: 5.2735 | Train Loss Components: Classification: 0.0381 | Classification_binary: 4.5983 | Regression_cos_phi: 0.0146 | Regression_energy: 0.2976 | Regression_eta: 0.0192 | Regression_pt: 0.2940 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7654 | Classification_binary: 4.5983 | Regression_cos_phi: 1.0277 | Regression_energy: 3.2610 | Regression_eta: 0.9441 | Regression_pt: 3.5251 | Regression_sin_phi: 0.7467 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.4438s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0915 | max_forward: 0.0770 | max_loss: 0.0030 | max_backward: 0.2021 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.2562 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2947.5 | uss_MB: 2142.8 | child_rss_MB: 4123.9 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:15:07,660] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:48,186] INFO : Step 400/50000 rank0 | Train Loss: 15.1895 | Train Loss Unweighted: 4.9275 | Train Loss Components: Classification: 0.0383 | Classification_binary: 4.2626 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2965 | Regression_eta: 0.0198 | Regression_pt: 0.2883 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7769 | Classification_binary: 4.2626 | Regression_cos_phi: 0.7469 | Regression_energy: 3.2491 | Regression_eta: 0.9717 | Regression_pt: 3.4568 | Regression_sin_phi: 0.7256 | LR: 1.00e-03 | DataLoad Time: 0.0019s | Model Forward Time: 1.4068s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1823 | max_loss: 0.0032 | max_backward: 0.2024 | max_optimizer_step: 0.1018 | max_tensorboard_logging: 0.2946 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 2947.0 | uss_MB: 2142.8 | child_rss_MB: 4121.4 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:15:48,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:30,174] INFO : Step 500/50000 rank0 | Train Loss: 15.0155 | Train Loss Unweighted: 4.9003 | Train Loss Components: Classification: 0.0386 | Classification_binary: 4.2703 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2761 | Regression_eta: 0.0220 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7927 | Classification_binary: 4.2703 | Regression_cos_phi: 0.7647 | Regression_energy: 3.0252 | Regression_eta: 1.0816 | Regression_pt: 3.2243 | Regression_sin_phi: 0.8568 | LR: 1.00e-03 | DataLoad Time: 0.0024s | Model Forward Time: 1.4569s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.1043 | max_loss: 0.0031 | max_backward: 0.3916 | max_optimizer_step: 0.1023 | max_tensorboard_logging: 0.2573 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2948.1 | uss_MB: 2144.9 | child_rss_MB: 4141.6 | allocated_MB: 295.1 | reserved_MB: 3800.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 11:16:30,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:11,141] INFO : Step 600/50000 rank0 | Train Loss: 15.6284 | Train Loss Unweighted: 4.8147 | Train Loss Components: Classification: 0.0357 | Classification_binary: 4.1430 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2976 | Regression_eta: 0.0373 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.6557 | Classification_binary: 4.1430 | Regression_cos_phi: 0.8000 | Regression_energy: 3.2613 | Regression_eta: 1.8338 | Regression_pt: 3.3660 | Regression_sin_phi: 0.5684 | LR: 1.00e-03 | DataLoad Time: 0.0023s | Model Forward Time: 1.3692s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3091 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2656 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 2883.6 | uss_MB: 2080.9 | child_rss_MB: 4138.6 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:17:11,352] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:51,965] INFO : Step 700/50000 rank0 | Train Loss: 14.7855 | Train Loss Unweighted: 4.9106 | Train Loss Components: Classification: 0.0371 | Classification_binary: 4.2729 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2808 | Regression_eta: 0.0175 | Regression_pt: 0.2797 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7223 | Classification_binary: 4.2729 | Regression_cos_phi: 0.7401 | Regression_energy: 3.0773 | Regression_eta: 0.8611 | Regression_pt: 3.3545 | Regression_sin_phi: 0.7574 | LR: 1.00e-03 | DataLoad Time: 0.0031s | Model Forward Time: 1.5770s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0963 | max_forward: 0.1040 | max_loss: 0.0032 | max_backward: 0.3922 | max_optimizer_step: 0.1029 | max_tensorboard_logging: 0.2810 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2886.5 | uss_MB: 2085.1 | child_rss_MB: 4115.4 | allocated_MB: 295.1 | reserved_MB: 3800.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:17:52,177] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:32,737] INFO : Step 800/50000 rank0 | Train Loss: 14.3978 | Train Loss Unweighted: 4.7712 | Train Loss Components: Classification: 0.0343 | Classification_binary: 4.1502 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2764 | Regression_eta: 0.0180 | Regression_pt: 0.2696 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5931 | Classification_binary: 4.1502 | Regression_cos_phi: 0.7676 | Regression_energy: 3.0284 | Regression_eta: 0.8860 | Regression_pt: 3.2333 | Regression_sin_phi: 0.7392 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2614s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.2022 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.2494 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2884.9 | uss_MB: 2083.5 | child_rss_MB: 4115.0 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:18:32,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:13,599] INFO : Step 900/50000 rank0 | Train Loss: 13.1860 | Train Loss Unweighted: 4.7561 | Train Loss Components: Classification: 0.0338 | Classification_binary: 4.2109 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2406 | Regression_eta: 0.0145 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5695 | Classification_binary: 4.2109 | Regression_cos_phi: 0.6346 | Regression_energy: 2.6367 | Regression_eta: 0.7112 | Regression_pt: 2.8582 | Regression_sin_phi: 0.5650 | LR: 9.99e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0711 | max_backward: 0.2024 | max_optimizer_step: 0.1010 | max_tensorboard_logging: 0.4631 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2884.9 | uss_MB: 2083.5 | child_rss_MB: 4115.6 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:19:13,812] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:54,516] INFO : Step 1000/50000 rank0 | Train Loss: 14.1952 | Train Loss Unweighted: 4.7871 | Train Loss Components: Classification: 0.0387 | Classification_binary: 4.2168 | Regression_cos_phi: 0.0121 | Regression_energy: 0.2518 | Regression_eta: 0.0195 | Regression_pt: 0.2359 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.7974 | Classification_binary: 4.2168 | Regression_cos_phi: 0.8521 | Regression_energy: 2.7586 | Regression_eta: 0.9598 | Regression_pt: 2.8287 | Regression_sin_phi: 0.7818 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3137s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1218 | max_loss: 0.0029 | max_backward: 0.2028 | max_optimizer_step: 0.1013 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2885.0 | uss_MB: 2083.6 | child_rss_MB: 4115.7 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:19:54,734] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:19:54,807] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:19:54,983] INFO : Running validation on rank0 +rank0 [2026-10-28 11:19:54,983] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:19:54,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:19:57,514] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:19:57,701] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:19:57,803] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:19:57,865] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:19:57,931] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:19:57,982] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:19:58,043] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:19:58,105] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:19:58,159] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:19:58,254] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:19:58,353] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:19:58,413] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:19:58,475] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:19:58,526] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:19:58,589] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:19:58,653] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:19:58,711] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:19:58,715] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:19:58,718] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:19:58,720] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:19:58,721] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:19:58,724] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:19:58,727] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:19:58,728] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:19:58,731] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:19:58,734] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:19:58,736] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:19:58,737] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:19:58,740] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:19:58,743] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:19:58,745] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:19:58,746] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:20:06,293] INFO : VALIDATION | Step=1000/50000 | Train Loss=14.1952 | Valid Loss=13.7567 | Train Loss Unweighted=4.7871 | Valid Loss Unweighted=4.7167 | Stale=0 | ETA=369.9m +rank0 [2026-10-28 11:20:06,300] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:06,361] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:20:06,371] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:06,506] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:08,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:09,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:10,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:10,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:10,817] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:20:10,820] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:10,875] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:10,878] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:10,963] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:12,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:13,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:14,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:14,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:14,897] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:20:14,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:14,954] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:14,957] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:20:15,099] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:16,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:17,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:18,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:18,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:18,986] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:18,994] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:19,253] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:20:19,802] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:21,656] INFO : Plotted jets +rank0 [2026-10-28 11:20:22,730] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:22,765] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:22,996] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:23,384] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:24,769] INFO : Plotted jets +rank0 [2026-10-28 11:20:25,870] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:25,904] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:26,110] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_1000/cld_edm_qq_pf +rank0 [2026-10-28 11:20:26,476] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:27,839] INFO : Plotted jets +rank0 [2026-10-28 11:20:28,874] INFO : Plotted jet ratio +rank0 [2026-10-28 11:21:09,928] INFO : Step 1100/50000 rank0 | Train Loss: 14.0203 | Train Loss Unweighted: 4.5956 | Train Loss Components: Classification: 0.0355 | Classification_binary: 4.0186 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2509 | Regression_eta: 0.0268 | Regression_pt: 0.2432 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.6471 | Classification_binary: 4.0186 | Regression_cos_phi: 0.7384 | Regression_energy: 2.7488 | Regression_eta: 1.3165 | Regression_pt: 2.9169 | Regression_sin_phi: 0.6339 | LR: 9.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4666s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0768 | max_loss: 0.0145 | max_backward: 0.2024 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3342 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3184.9 | uss_MB: 2392.5 | child_rss_MB: 17755.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:21:10,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:51,081] INFO : Step 1200/50000 rank0 | Train Loss: 13.5316 | Train Loss Unweighted: 4.5611 | Train Loss Components: Classification: 0.0342 | Classification_binary: 4.0019 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2466 | Regression_eta: 0.0192 | Regression_pt: 0.2374 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5889 | Classification_binary: 4.0019 | Regression_cos_phi: 0.7376 | Regression_energy: 2.7016 | Regression_eta: 0.9430 | Regression_pt: 2.8470 | Regression_sin_phi: 0.7115 | LR: 9.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4305s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0522 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.2641 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3184.2 | uss_MB: 2392.9 | child_rss_MB: 4111.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:21:51,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:33,381] INFO : Step 1300/50000 rank0 | Train Loss: 14.4450 | Train Loss Unweighted: 4.6261 | Train Loss Components: Classification: 0.0328 | Classification_binary: 4.0426 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2520 | Regression_eta: 0.0326 | Regression_pt: 0.2420 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5206 | Classification_binary: 4.0426 | Regression_cos_phi: 0.8621 | Regression_energy: 2.7613 | Regression_eta: 1.6017 | Regression_pt: 2.9023 | Regression_sin_phi: 0.7545 | LR: 9.98e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6043s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1758 | max_loss: 0.0030 | max_backward: 0.2868 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 0.3130 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3185.8 | uss_MB: 2394.8 | child_rss_MB: 4107.7 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:22:33,613] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:14,337] INFO : Step 1400/50000 rank0 | Train Loss: 13.5429 | Train Loss Unweighted: 4.5340 | Train Loss Components: Classification: 0.0318 | Classification_binary: 3.9808 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2429 | Regression_eta: 0.0247 | Regression_pt: 0.2322 | Regression_sin_phi: 0.0127 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4767 | Classification_binary: 3.9808 | Regression_cos_phi: 0.6242 | Regression_energy: 2.6612 | Regression_eta: 1.2133 | Regression_pt: 2.7849 | Regression_sin_phi: 0.8019 | LR: 9.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4952s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0720 | max_forward: 0.0774 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.2882 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3182.6 | uss_MB: 2391.4 | child_rss_MB: 4109.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:23:14,566] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:55,775] INFO : Step 1500/50000 rank0 | Train Loss: 13.0110 | Train Loss Unweighted: 4.5981 | Train Loss Components: Classification: 0.0317 | Classification_binary: 4.0673 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2360 | Regression_eta: 0.0184 | Regression_pt: 0.2255 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4689 | Classification_binary: 4.0673 | Regression_cos_phi: 0.6535 | Regression_energy: 2.5859 | Regression_eta: 0.9058 | Regression_pt: 2.7043 | Regression_sin_phi: 0.6252 | LR: 9.98e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4262s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1843 | max_loss: 0.0031 | max_backward: 0.2027 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.2826 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3182.5 | uss_MB: 2391.4 | child_rss_MB: 4106.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:23:55,986] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:36,718] INFO : Step 1600/50000 rank0 | Train Loss: 13.4222 | Train Loss Unweighted: 4.5041 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.9498 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2433 | Regression_eta: 0.0220 | Regression_pt: 0.2359 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4711 | Classification_binary: 3.9498 | Regression_cos_phi: 0.7589 | Regression_energy: 2.6664 | Regression_eta: 1.0816 | Regression_pt: 2.8294 | Regression_sin_phi: 0.6649 | LR: 9.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4503s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1839 | max_loss: 0.0030 | max_backward: 0.2025 | max_optimizer_step: 0.1073 | max_tensorboard_logging: 0.2999 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3043.7 | uss_MB: 2253.1 | child_rss_MB: 4105.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:24:36,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:17,499] INFO : Step 1700/50000 rank0 | Train Loss: 12.2893 | Train Loss Unweighted: 4.4091 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.9110 | Regression_cos_phi: 0.0076 | Regression_energy: 0.2217 | Regression_eta: 0.0190 | Regression_pt: 0.2102 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5035 | Classification_binary: 3.9110 | Regression_cos_phi: 0.5340 | Regression_energy: 2.4292 | Regression_eta: 0.9321 | Regression_pt: 2.5210 | Regression_sin_phi: 0.4585 | LR: 9.97e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4236s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0771 | max_loss: 0.0028 | max_backward: 0.3109 | max_optimizer_step: 0.1055 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3044.7 | uss_MB: 2255.2 | child_rss_MB: 4107.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:25:17,713] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:57,330] INFO : Step 1800/50000 rank0 | Train Loss: 12.9007 | Train Loss Unweighted: 4.5419 | Train Loss Components: Classification: 0.0326 | Classification_binary: 4.0293 | Regression_cos_phi: 0.0099 | Regression_energy: 0.2226 | Regression_eta: 0.0203 | Regression_pt: 0.2174 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5146 | Classification_binary: 4.0293 | Regression_cos_phi: 0.6991 | Regression_energy: 2.4392 | Regression_eta: 0.9992 | Regression_pt: 2.6066 | Regression_sin_phi: 0.6127 | LR: 9.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5004s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.3076 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.3799 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3045.3 | uss_MB: 2255.3 | child_rss_MB: 4101.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:25:57,544] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:38,300] INFO : Step 1900/50000 rank0 | Train Loss: 12.1641 | Train Loss Unweighted: 4.3945 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.8891 | Regression_cos_phi: 0.0086 | Regression_energy: 0.2242 | Regression_eta: 0.0123 | Regression_pt: 0.2214 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3955 | Classification_binary: 3.8891 | Regression_cos_phi: 0.6042 | Regression_energy: 2.4568 | Regression_eta: 0.6056 | Regression_pt: 2.6549 | Regression_sin_phi: 0.5580 | LR: 9.97e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.6177s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1042 | max_loss: 0.0031 | max_backward: 0.3865 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3047.1 | uss_MB: 2257.8 | child_rss_MB: 4098.4 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:26:38,509] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:19,021] INFO : Step 2000/50000 rank0 | Train Loss: 13.0496 | Train Loss Unweighted: 4.4408 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.8727 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2549 | Regression_eta: 0.0148 | Regression_pt: 0.2484 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5202 | Classification_binary: 3.8727 | Regression_cos_phi: 0.6327 | Regression_energy: 2.7932 | Regression_eta: 0.7266 | Regression_pt: 2.9786 | Regression_sin_phi: 0.5256 | LR: 9.96e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4105s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3107 | max_optimizer_step: 0.1046 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3044.9 | uss_MB: 2256.6 | child_rss_MB: 4101.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:27:19,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:19,252] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:27:19,425] INFO : Running validation on rank0 +rank0 [2026-10-28 11:27:19,425] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:27:19,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:20,741] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:27:20,885] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:21,016] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:21,091] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:21,161] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:21,229] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:21,298] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:21,367] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:21,469] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:21,600] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:21,731] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:21,801] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:21,869] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:21,938] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:22,007] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:22,076] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:22,177] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:22,184] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:22,187] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:22,190] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:22,193] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:22,195] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:22,197] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:22,199] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:22,203] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:22,206] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:22,207] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:22,209] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:22,212] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:22,213] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:22,216] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:22,218] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:29,914] INFO : VALIDATION | Step=2000/50000 | Train Loss=13.0496 | Valid Loss=12.6006 | Train Loss Unweighted=4.4408 | Valid Loss Unweighted=4.4343 | Stale=0 | ETA=358.6m +rank0 [2026-10-28 11:27:29,915] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:29,973] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:29,977] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:30,115] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:32,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:32,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:33,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:34,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:34,250] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:27:34,253] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:34,309] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:34,312] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:34,390] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:36,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:36,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:37,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:38,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:38,344] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:27:38,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:38,401] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:38,404] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:27:38,556] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:40,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:41,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:41,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:42,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:42,437] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:42,445] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:42,700] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:27:43,102] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:44,943] INFO : Plotted jets +rank0 [2026-10-28 11:27:46,024] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:46,056] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:46,283] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:46,676] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:48,070] INFO : Plotted jets +rank0 [2026-10-28 11:27:49,174] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:49,208] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:49,408] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_2000/cld_edm_qq_pf +rank0 [2026-10-28 11:27:49,771] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:51,144] INFO : Plotted jets +rank0 [2026-10-28 11:27:52,187] INFO : Plotted jet ratio +rank0 [2026-10-28 11:28:33,384] INFO : Step 2100/50000 rank0 | Train Loss: 13.3832 | Train Loss Unweighted: 4.4776 | Train Loss Components: Classification: 0.0325 | Classification_binary: 3.9215 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2467 | Regression_eta: 0.0188 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5079 | Classification_binary: 3.9215 | Regression_cos_phi: 0.7744 | Regression_energy: 2.7032 | Regression_eta: 0.9249 | Regression_pt: 2.8247 | Regression_sin_phi: 0.7266 | LR: 9.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1856 | max_loss: 0.0030 | max_backward: 0.2029 | max_optimizer_step: 0.1100 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3281.3 | uss_MB: 2495.3 | child_rss_MB: 4047.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:28:33,597] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:14,317] INFO : Step 2200/50000 rank0 | Train Loss: 11.9067 | Train Loss Unweighted: 4.3015 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.8337 | Regression_cos_phi: 0.0097 | Regression_energy: 0.2070 | Regression_eta: 0.0160 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.5029 | Classification_binary: 3.8337 | Regression_cos_phi: 0.6858 | Regression_energy: 2.2677 | Regression_eta: 0.7876 | Regression_pt: 2.3368 | Regression_sin_phi: 0.4921 | LR: 9.96e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1038 | max_loss: 0.0031 | max_backward: 0.3373 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3858 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3282.9 | uss_MB: 2497.4 | child_rss_MB: 4049.0 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:29:14,529] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:55,853] INFO : Step 2300/50000 rank0 | Train Loss: 12.2761 | Train Loss Unweighted: 4.4087 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.8831 | Regression_cos_phi: 0.0068 | Regression_energy: 0.2374 | Regression_eta: 0.0154 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3783 | Classification_binary: 3.8831 | Regression_cos_phi: 0.4780 | Regression_energy: 2.6016 | Regression_eta: 0.7567 | Regression_pt: 2.7521 | Regression_sin_phi: 0.4263 | LR: 9.95e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4669s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1822 | max_loss: 0.0030 | max_backward: 0.2023 | max_optimizer_step: 0.1074 | max_tensorboard_logging: 0.3016 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3282.3 | uss_MB: 2495.4 | child_rss_MB: 4042.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:29:56,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:37,466] INFO : Step 2400/50000 rank0 | Train Loss: 12.4005 | Train Loss Unweighted: 4.3180 | Train Loss Components: Classification: 0.0290 | Classification_binary: 3.8035 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2286 | Regression_eta: 0.0145 | Regression_pt: 0.2217 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3449 | Classification_binary: 3.8035 | Regression_cos_phi: 0.6727 | Regression_energy: 2.5044 | Regression_eta: 0.7148 | Regression_pt: 2.6588 | Regression_sin_phi: 0.7015 | LR: 9.95e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4109s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1844 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1074 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3282.3 | uss_MB: 2495.4 | child_rss_MB: 4043.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:30:37,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:18,295] INFO : Step 2500/50000 rank0 | Train Loss: 11.5982 | Train Loss Unweighted: 4.1917 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.7021 | Regression_cos_phi: 0.0083 | Regression_energy: 0.2178 | Regression_eta: 0.0105 | Regression_pt: 0.2166 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3275 | Classification_binary: 3.7021 | Regression_cos_phi: 0.5857 | Regression_energy: 2.3867 | Regression_eta: 0.5172 | Regression_pt: 2.5980 | Regression_sin_phi: 0.4809 | LR: 9.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4622s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0767 | max_loss: 0.0029 | max_backward: 0.2284 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3822 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3282.3 | uss_MB: 2495.4 | child_rss_MB: 4042.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:31:18,507] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:00,037] INFO : Step 2600/50000 rank0 | Train Loss: 11.0203 | Train Loss Unweighted: 4.1058 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.6802 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1851 | Regression_eta: 0.0110 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3677 | Classification_binary: 3.6802 | Regression_cos_phi: 0.5588 | Regression_energy: 2.0280 | Regression_eta: 0.5413 | Regression_pt: 2.1767 | Regression_sin_phi: 0.6676 | LR: 9.94e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6175s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1041 | max_loss: 0.0030 | max_backward: 0.3429 | max_optimizer_step: 0.1071 | max_tensorboard_logging: 0.4077 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.2 | uss_MB: 2332.4 | child_rss_MB: 4042.4 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:32:00,252] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:40,998] INFO : Step 2700/50000 rank0 | Train Loss: 12.4271 | Train Loss Unweighted: 4.3367 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.7987 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2402 | Regression_eta: 0.0114 | Regression_pt: 0.2391 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3551 | Classification_binary: 3.7987 | Regression_cos_phi: 0.6738 | Regression_energy: 2.6319 | Regression_eta: 0.5583 | Regression_pt: 2.8672 | Regression_sin_phi: 0.5422 | LR: 9.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3870s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1323 | max_loss: 0.0029 | max_backward: 0.2032 | max_optimizer_step: 0.1076 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.2 | uss_MB: 2331.8 | child_rss_MB: 4043.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:32:41,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:21,453] INFO : Step 2800/50000 rank0 | Train Loss: 11.5494 | Train Loss Unweighted: 4.2442 | Train Loss Components: Classification: 0.0290 | Classification_binary: 3.8007 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1928 | Regression_eta: 0.0170 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3465 | Classification_binary: 3.8007 | Regression_cos_phi: 0.6187 | Regression_energy: 2.1125 | Regression_eta: 0.8348 | Regression_pt: 2.2366 | Regression_sin_phi: 0.5995 | LR: 9.93e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4697s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0543 | max_forward: 0.0769 | max_loss: 0.0031 | max_backward: 0.2019 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.2848 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3118.2 | uss_MB: 2332.4 | child_rss_MB: 4039.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:33:21,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:01,729] INFO : Step 2900/50000 rank0 | Train Loss: 10.9168 | Train Loss Unweighted: 4.1168 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.6592 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2046 | Regression_eta: 0.0090 | Regression_pt: 0.2035 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2934 | Classification_binary: 3.6592 | Regression_cos_phi: 0.4482 | Regression_energy: 2.2422 | Regression_eta: 0.4441 | Regression_pt: 2.4407 | Regression_sin_phi: 0.3891 | LR: 9.93e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5006s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0842 | max_forward: 0.0776 | max_loss: 0.0029 | max_backward: 0.2025 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.2788 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.2 | uss_MB: 2332.6 | child_rss_MB: 4039.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:34:01,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:42,395] INFO : Step 3000/50000 rank0 | Train Loss: 11.9253 | Train Loss Unweighted: 4.1456 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.6636 | Regression_cos_phi: 0.0118 | Regression_energy: 0.2114 | Regression_eta: 0.0152 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3161 | Classification_binary: 3.6636 | Regression_cos_phi: 0.8298 | Regression_energy: 2.3165 | Regression_eta: 0.7459 | Regression_pt: 2.4707 | Regression_sin_phi: 0.5826 | LR: 9.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5329s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1034 | max_loss: 0.0556 | max_backward: 0.2863 | max_optimizer_step: 0.1064 | max_tensorboard_logging: 0.3339 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3118.7 | uss_MB: 2335.5 | child_rss_MB: 4039.5 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:34:42,609] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:42,660] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:34:42,835] INFO : Running validation on rank0 +rank0 [2026-10-28 11:34:42,835] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:34:42,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:34:44,155] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:34:44,299] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:34:44,430] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:34:44,504] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:34:44,574] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:34:44,642] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:34:44,712] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:34:44,780] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:34:44,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:34:45,012] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:34:45,143] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:34:45,214] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:34:45,282] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:34:45,350] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:34:45,418] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:34:45,487] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:34:45,604] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:34:45,612] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:34:45,614] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:34:45,617] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:34:45,618] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:34:45,621] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:34:45,623] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:34:45,625] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:34:45,628] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:34:45,631] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:34:45,633] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:34:45,636] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:34:45,637] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:34:45,640] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:34:45,642] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:34:45,644] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:34:53,157] INFO : VALIDATION | Step=3000/50000 | Train Loss=11.9253 | Valid Loss=11.6295 | Train Loss Unweighted=4.1456 | Valid Loss Unweighted=4.1618 | Stale=0 | ETA=349.8m +rank0 [2026-10-28 11:34:53,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:53,230] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:34:53,233] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:34:53,377] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:55,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:55,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:56,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:57,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:57,385] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:34:57,388] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:57,443] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:34:57,446] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:57,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:59,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:35:00,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:35:00,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:35:01,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:35:01,486] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:35:01,490] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:35:01,544] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:35:01,547] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:35:01,633] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:35:03,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:35:04,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:35:04,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:35:05,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:35:05,561] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:35:05,569] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:35:05,826] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:35:06,229] INFO : Plotted number of elements +rank0 [2026-10-28 11:35:08,090] INFO : Plotted jets +rank0 [2026-10-28 11:35:09,170] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:09,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:35:09,431] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:35:09,819] INFO : Plotted number of elements +rank0 [2026-10-28 11:35:11,210] INFO : Plotted jets +rank0 [2026-10-28 11:35:12,318] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:12,351] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:35:12,554] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_3000/cld_edm_qq_pf +rank0 [2026-10-28 11:35:12,915] INFO : Plotted number of elements +rank0 [2026-10-28 11:35:14,287] INFO : Plotted jets +rank0 [2026-10-28 11:35:15,718] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:57,289] INFO : Step 3100/50000 rank0 | Train Loss: 12.0174 | Train Loss Unweighted: 4.2579 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.7798 | Regression_cos_phi: 0.0086 | Regression_energy: 0.2099 | Regression_eta: 0.0186 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3918 | Classification_binary: 3.7798 | Regression_cos_phi: 0.6035 | Regression_energy: 2.3003 | Regression_eta: 0.9140 | Regression_pt: 2.4145 | Regression_sin_phi: 0.6136 | LR: 9.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.3110 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.3877 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4037.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:35:57,500] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:38,300] INFO : Step 3200/50000 rank0 | Train Loss: 11.1021 | Train Loss Unweighted: 4.0789 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.6534 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1822 | Regression_eta: 0.0148 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.4012 | Classification_binary: 3.6534 | Regression_cos_phi: 0.6408 | Regression_energy: 1.9959 | Regression_eta: 0.7300 | Regression_pt: 2.1717 | Regression_sin_phi: 0.5092 | LR: 9.91e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4908s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0626 | max_backward: 0.2029 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4038.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 11:36:38,511] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:19,121] INFO : Step 3300/50000 rank0 | Train Loss: 12.4079 | Train Loss Unweighted: 4.2953 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.7670 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2343 | Regression_eta: 0.0141 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3848 | Classification_binary: 3.7670 | Regression_cos_phi: 0.6463 | Regression_energy: 2.5669 | Regression_eta: 0.6915 | Regression_pt: 2.7804 | Regression_sin_phi: 0.5709 | LR: 9.90e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0767 | max_loss: 0.0030 | max_backward: 0.2743 | max_optimizer_step: 0.1086 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4034.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:37:19,332] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:00,624] INFO : Step 3400/50000 rank0 | Train Loss: 12.2776 | Train Loss Unweighted: 4.2387 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.7132 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2370 | Regression_eta: 0.0168 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2652 | Classification_binary: 3.7132 | Regression_cos_phi: 0.6471 | Regression_energy: 2.5969 | Regression_eta: 0.8245 | Regression_pt: 2.7242 | Regression_sin_phi: 0.5066 | LR: 9.90e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5363s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.3107 | max_optimizer_step: 0.1078 | max_tensorboard_logging: 0.3765 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3339.1 | uss_MB: 2557.0 | child_rss_MB: 4035.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:38:00,836] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:41,625] INFO : Step 3500/50000 rank0 | Train Loss: 11.4570 | Train Loss Unweighted: 4.0172 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.5304 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2177 | Regression_eta: 0.0108 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3115 | Classification_binary: 3.5304 | Regression_cos_phi: 0.6490 | Regression_energy: 2.3857 | Regression_eta: 0.5291 | Regression_pt: 2.5541 | Regression_sin_phi: 0.4971 | LR: 9.89e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5616s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0032 | max_backward: 0.3069 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.3847 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3339.1 | uss_MB: 2556.9 | child_rss_MB: 4035.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:38:41,837] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:21,940] INFO : Step 3600/50000 rank0 | Train Loss: 11.1187 | Train Loss Unweighted: 3.9992 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.5823 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1781 | Regression_eta: 0.0213 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3090 | Classification_binary: 3.5823 | Regression_cos_phi: 0.6056 | Regression_energy: 1.9519 | Regression_eta: 1.0490 | Regression_pt: 2.0596 | Regression_sin_phi: 0.5613 | LR: 9.89e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0030 | max_backward: 0.2324 | max_optimizer_step: 0.1094 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3190.3 | uss_MB: 2406.6 | child_rss_MB: 4034.0 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:39:22,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:03,621] INFO : Step 3700/50000 rank0 | Train Loss: 12.8052 | Train Loss Unweighted: 3.9794 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.4605 | Regression_cos_phi: 0.0095 | Regression_energy: 0.2274 | Regression_eta: 0.0350 | Regression_pt: 0.2083 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3185 | Classification_binary: 3.4605 | Regression_cos_phi: 0.6658 | Regression_energy: 2.4914 | Regression_eta: 1.7199 | Regression_pt: 2.4980 | Regression_sin_phi: 0.6513 | LR: 9.88e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4793s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0371 | max_backward: 0.2031 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3884 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3192.3 | uss_MB: 2408.2 | child_rss_MB: 4027.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:40:03,834] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:44,689] INFO : Step 3800/50000 rank0 | Train Loss: 10.8068 | Train Loss Unweighted: 4.0242 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.5899 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1910 | Regression_eta: 0.0116 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2185 | Classification_binary: 3.5899 | Regression_cos_phi: 0.5888 | Regression_energy: 2.0931 | Regression_eta: 0.5686 | Regression_pt: 2.2732 | Regression_sin_phi: 0.4747 | LR: 9.87e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4594s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0501 | max_forward: 0.0770 | max_loss: 0.0037 | max_backward: 0.2025 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3192.3 | uss_MB: 2408.9 | child_rss_MB: 4030.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:40:44,913] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:26,709] INFO : Step 3900/50000 rank0 | Train Loss: 11.5790 | Train Loss Unweighted: 3.9766 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.5032 | Regression_cos_phi: 0.0080 | Regression_energy: 0.2088 | Regression_eta: 0.0200 | Regression_pt: 0.2002 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2782 | Classification_binary: 3.5032 | Regression_cos_phi: 0.5600 | Regression_energy: 2.2875 | Regression_eta: 0.9818 | Regression_pt: 2.4009 | Regression_sin_phi: 0.5673 | LR: 9.87e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4467s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1835 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.3085 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3192.3 | uss_MB: 2409.3 | child_rss_MB: 4030.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:41:26,920] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:07,850] INFO : Step 4000/50000 rank0 | Train Loss: 10.8825 | Train Loss Unweighted: 3.9858 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.5208 | Regression_cos_phi: 0.0070 | Regression_energy: 0.2084 | Regression_eta: 0.0090 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1961 | Classification_binary: 3.5208 | Regression_cos_phi: 0.4957 | Regression_energy: 2.2830 | Regression_eta: 0.4404 | Regression_pt: 2.4894 | Regression_sin_phi: 0.4572 | LR: 9.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4754s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0681 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2805 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3192.3 | uss_MB: 2409.3 | child_rss_MB: 4031.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:42:08,080] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:08,100] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:42:08,275] INFO : Running validation on rank0 +rank0 [2026-10-28 11:42:08,275] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:42:08,275] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:42:09,594] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:42:09,737] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:42:09,868] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:42:09,942] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:42:10,010] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:42:10,081] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:42:10,151] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:42:10,219] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:42:10,323] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:42:10,452] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:42:10,583] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:42:10,654] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:42:10,723] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:42:10,791] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:42:10,859] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:42:10,927] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:42:11,046] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:42:11,053] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:42:11,057] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:42:11,059] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:42:11,060] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:42:11,063] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:42:11,066] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:42:11,069] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:42:11,069] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:42:11,072] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:42:11,075] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:42:11,077] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:42:11,079] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:42:11,080] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:42:11,083] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:42:11,086] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:42:18,802] INFO : VALIDATION | Step=4000/50000 | Train Loss=10.8825 | Valid Loss=11.0341 | Train Loss Unweighted=3.9858 | Valid Loss Unweighted=3.9445 | Stale=0 | ETA=342.2m +rank0 [2026-10-28 11:42:18,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:42:18,877] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:42:18,881] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:42:18,969] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:42:20,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:42:21,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:42:22,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:42:22,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:42:22,727] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:42:22,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:42:22,785] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:42:22,788] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:42:22,864] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:42:24,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:42:25,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:42:25,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:42:26,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:42:26,646] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:42:26,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:42:26,704] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:42:26,707] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:42:26,856] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:42:28,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:42:29,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:42:29,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:42:30,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:42:30,762] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:42:30,769] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:42:31,028] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:42:31,434] INFO : Plotted number of elements +rank0 [2026-10-28 11:42:33,337] INFO : Plotted jets +rank0 [2026-10-28 11:42:34,418] INFO : Plotted jet ratio +rank0 [2026-10-28 11:42:34,449] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:42:34,676] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:42:35,062] INFO : Plotted number of elements +rank0 [2026-10-28 11:42:36,450] INFO : Plotted jets +rank0 [2026-10-28 11:42:37,548] INFO : Plotted jet ratio +rank0 [2026-10-28 11:42:37,580] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:42:37,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_4000/cld_edm_qq_pf +rank0 [2026-10-28 11:42:38,148] INFO : Plotted number of elements +rank0 [2026-10-28 11:42:39,509] INFO : Plotted jets +rank0 [2026-10-28 11:42:40,982] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:22,124] INFO : Step 4100/50000 rank0 | Train Loss: 11.0773 | Train Loss Unweighted: 3.9121 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.4785 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1896 | Regression_eta: 0.0220 | Regression_pt: 0.1804 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1438 | Classification_binary: 3.4785 | Regression_cos_phi: 0.5597 | Regression_energy: 2.0770 | Regression_eta: 1.0808 | Regression_pt: 2.1628 | Regression_sin_phi: 0.5746 | LR: 9.85e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4603s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1841 | max_loss: 0.0033 | max_backward: 0.2021 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3397.6 | uss_MB: 2617.1 | child_rss_MB: 4026.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:43:22,341] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:02,522] INFO : Step 4200/50000 rank0 | Train Loss: 10.8108 | Train Loss Unweighted: 3.8229 | Train Loss Components: Classification: 0.0281 | Classification_binary: 3.4045 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1835 | Regression_eta: 0.0153 | Regression_pt: 0.1726 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3033 | Classification_binary: 3.4045 | Regression_cos_phi: 0.7144 | Regression_energy: 2.0111 | Regression_eta: 0.7508 | Regression_pt: 2.0695 | Regression_sin_phi: 0.5573 | LR: 9.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4730s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0272 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.2033 | max_optimizer_step: 0.1852 | max_tensorboard_logging: 0.2914 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4024.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 11:44:02,754] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:43,848] INFO : Step 4300/50000 rank0 | Train Loss: 10.6818 | Train Loss Unweighted: 3.7630 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.3376 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1858 | Regression_eta: 0.0190 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2174 | Classification_binary: 3.3376 | Regression_cos_phi: 0.5066 | Regression_energy: 2.0357 | Regression_eta: 0.9363 | Regression_pt: 2.1495 | Regression_sin_phi: 0.4988 | LR: 9.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4591s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1700 | max_loss: 0.0030 | max_backward: 0.2020 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3013 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4020.4 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 11:44:44,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:24,902] INFO : Step 4400/50000 rank0 | Train Loss: 10.2831 | Train Loss Unweighted: 3.8156 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.4102 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1775 | Regression_eta: 0.0104 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1185 | Classification_binary: 3.4102 | Regression_cos_phi: 0.6066 | Regression_energy: 1.9454 | Regression_eta: 0.5097 | Regression_pt: 2.1042 | Regression_sin_phi: 0.5884 | LR: 9.83e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6632s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0932 | max_forward: 0.1044 | max_loss: 0.0031 | max_backward: 0.3922 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0123 | Memory: rss_MB: 3397.6 | uss_MB: 2619.2 | child_rss_MB: 4047.5 | allocated_MB: 295.1 | reserved_MB: 3896.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 11:45:25,115] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:05,711] INFO : Step 4500/50000 rank0 | Train Loss: 10.7525 | Train Loss Unweighted: 3.6803 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.2673 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1796 | Regression_eta: 0.0166 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1778 | Classification_binary: 3.2673 | Regression_cos_phi: 0.8810 | Regression_energy: 1.9683 | Regression_eta: 0.8173 | Regression_pt: 2.0278 | Regression_sin_phi: 0.6130 | LR: 9.82e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4609s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0011 | max_forward: 0.0768 | max_loss: 0.0030 | max_backward: 0.3074 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.2948 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4048.0 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:46:05,924] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:46,758] INFO : Step 4600/50000 rank0 | Train Loss: 10.5031 | Train Loss Unweighted: 3.8296 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.4190 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1822 | Regression_eta: 0.0143 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2442 | Classification_binary: 3.4190 | Regression_cos_phi: 0.6101 | Regression_energy: 1.9964 | Regression_eta: 0.7033 | Regression_pt: 2.0508 | Regression_sin_phi: 0.4794 | LR: 9.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5928s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3087 | max_optimizer_step: 0.1124 | max_tensorboard_logging: 0.3965 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3397.6 | uss_MB: 2617.2 | child_rss_MB: 4025.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:46:46,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:27,461] INFO : Step 4700/50000 rank0 | Train Loss: 10.8839 | Train Loss Unweighted: 3.7758 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.3581 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1780 | Regression_eta: 0.0237 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2341 | Classification_binary: 3.3581 | Regression_cos_phi: 0.5954 | Regression_energy: 1.9507 | Regression_eta: 1.1656 | Regression_pt: 2.0727 | Regression_sin_phi: 0.5073 | LR: 9.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4757s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0769 | max_loss: 0.0030 | max_backward: 0.3092 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.3011 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3262.4 | uss_MB: 2481.6 | child_rss_MB: 4046.1 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 11:47:27,672] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:08,075] INFO : Step 4800/50000 rank0 | Train Loss: 11.8103 | Train Loss Unweighted: 3.9279 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.4434 | Regression_cos_phi: 0.0099 | Regression_energy: 0.2122 | Regression_eta: 0.0176 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.3693 | Classification_binary: 3.4434 | Regression_cos_phi: 0.6991 | Regression_energy: 2.3245 | Regression_eta: 0.8653 | Regression_pt: 2.4586 | Regression_sin_phi: 0.6501 | LR: 9.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4230s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.1338 | max_loss: 0.0032 | max_backward: 0.2022 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.3073 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3262.4 | uss_MB: 2481.8 | child_rss_MB: 4019.8 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:48:08,290] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:48,903] INFO : Step 4900/50000 rank0 | Train Loss: 10.7401 | Train Loss Unweighted: 3.7777 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.3387 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1931 | Regression_eta: 0.0136 | Regression_pt: 0.1890 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.2334 | Classification_binary: 3.3387 | Regression_cos_phi: 0.6055 | Regression_energy: 2.1160 | Regression_eta: 0.6675 | Regression_pt: 2.2663 | Regression_sin_phi: 0.5126 | LR: 9.79e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5204s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0769 | max_loss: 0.0857 | max_backward: 0.2024 | max_optimizer_step: 0.1063 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3262.4 | uss_MB: 2481.9 | child_rss_MB: 4018.9 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 11:48:49,116] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:29,750] INFO : Step 5000/50000 rank0 | Train Loss: 9.5813 | Train Loss Unweighted: 3.5707 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.1867 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1687 | Regression_eta: 0.0098 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1676 | Classification_binary: 3.1867 | Regression_cos_phi: 0.4537 | Regression_energy: 1.8484 | Regression_eta: 0.4822 | Regression_pt: 2.0014 | Regression_sin_phi: 0.4414 | LR: 9.78e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4425s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0031 | max_backward: 0.3081 | max_optimizer_step: 0.1060 | max_tensorboard_logging: 0.3052 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3262.4 | uss_MB: 2482.0 | child_rss_MB: 4046.3 | allocated_MB: 291.4 | reserved_MB: 3896.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:49:29,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:29,984] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:49:30,156] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:30,156] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:30,156] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:49:31,377] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:49:31,522] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:49:31,654] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:49:31,728] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:49:31,797] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:49:31,865] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:49:31,935] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:49:32,004] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:49:32,107] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:49:32,238] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:49:32,368] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:49:32,439] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:49:32,507] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:49:32,576] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:49:32,645] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:49:32,714] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:49:32,817] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:49:32,824] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:49:32,826] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:49:32,829] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:49:32,832] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:49:32,833] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:49:32,836] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:49:32,839] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:49:32,840] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:49:32,843] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:49:32,845] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:49:32,846] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:49:32,849] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:49:32,850] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:49:32,853] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:49:32,856] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:49:40,462] INFO : VALIDATION | Step=5000/50000 | Train Loss=9.5813 | Valid Loss=10.5400 | Train Loss Unweighted=3.5707 | Valid Loss Unweighted=3.7617 | Stale=0 | ETA=334.1m +rank0 [2026-10-28 11:49:40,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:40,529] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:49:40,533] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:40,680] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:42,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:43,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:44,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:44,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:44,841] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:49:44,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:44,908] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:49:44,911] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:44,999] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:46,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:47,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:48,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:48,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:48,939] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:49:48,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:48,996] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:49:48,999] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:49:49,149] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:50,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:51,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:52,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:52,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:53,022] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:49:53,030] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:53,288] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:49:53,690] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:55,587] INFO : Plotted jets +rank0 [2026-10-28 11:49:56,668] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:56,702] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:56,932] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:57,320] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:58,713] INFO : Plotted jets +rank0 [2026-10-28 11:49:59,816] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:59,851] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:00,060] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_5000/cld_edm_qq_pf +rank0 [2026-10-28 11:50:00,420] INFO : Plotted number of elements +rank0 [2026-10-28 11:50:01,789] INFO : Plotted jets +rank0 [2026-10-28 11:50:02,833] INFO : Plotted jet ratio +rank0 [2026-10-28 11:50:43,642] INFO : Step 5100/50000 rank0 | Train Loss: 10.8364 | Train Loss Unweighted: 3.8688 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.4340 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1899 | Regression_eta: 0.0205 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1771 | Classification_binary: 3.4340 | Regression_cos_phi: 0.4539 | Regression_energy: 2.0810 | Regression_eta: 1.0092 | Regression_pt: 2.2213 | Regression_sin_phi: 0.4599 | LR: 9.77e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5826s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0030 | max_backward: 0.3092 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.4015 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3496.5 | uss_MB: 2718.6 | child_rss_MB: 4022.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:50:43,856] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:25,294] INFO : Step 5200/50000 rank0 | Train Loss: 10.6593 | Train Loss Unweighted: 3.7948 | Train Loss Components: Classification: 0.0238 | Classification_binary: 3.3521 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1983 | Regression_eta: 0.0136 | Regression_pt: 0.1909 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1062 | Classification_binary: 3.3521 | Regression_cos_phi: 0.5596 | Regression_energy: 2.1727 | Regression_eta: 0.6687 | Regression_pt: 2.2888 | Regression_sin_phi: 0.5113 | LR: 9.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7087s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1046 | max_loss: 0.0031 | max_backward: 0.3940 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3958 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3496.5 | uss_MB: 2720.8 | child_rss_MB: 4024.3 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:51:25,509] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:08,326] INFO : Step 5300/50000 rank0 | Train Loss: 10.0252 | Train Loss Unweighted: 3.5848 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.1807 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1791 | Regression_eta: 0.0127 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1755 | Classification_binary: 3.1807 | Regression_cos_phi: 0.5382 | Regression_energy: 1.9627 | Regression_eta: 0.6240 | Regression_pt: 2.0582 | Regression_sin_phi: 0.4859 | LR: 9.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5256s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1829 | max_loss: 0.0030 | max_backward: 0.2028 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3133 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4024.0 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:52:08,540] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:48,496] INFO : Step 5400/50000 rank0 | Train Loss: 10.6395 | Train Loss Unweighted: 3.5770 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.1260 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1995 | Regression_eta: 0.0168 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1187 | Classification_binary: 3.1260 | Regression_cos_phi: 0.5563 | Regression_energy: 2.1856 | Regression_eta: 0.8254 | Regression_pt: 2.3394 | Regression_sin_phi: 0.4882 | LR: 9.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4228s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1384 | max_loss: 0.0033 | max_backward: 0.2034 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4025.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:52:48,711] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:29,905] INFO : Step 5500/50000 rank0 | Train Loss: 10.4706 | Train Loss Unweighted: 3.5341 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1035 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1965 | Regression_eta: 0.0185 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1222 | Classification_binary: 3.1035 | Regression_cos_phi: 0.6030 | Regression_energy: 2.1528 | Regression_eta: 0.9077 | Regression_pt: 2.1024 | Regression_sin_phi: 0.4789 | LR: 9.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5663s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3082 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3847 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4021.1 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:53:30,118] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:10,887] INFO : Step 5600/50000 rank0 | Train Loss: 9.8073 | Train Loss Unweighted: 3.6007 | Train Loss Components: Classification: 0.0230 | Classification_binary: 3.2074 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1708 | Regression_eta: 0.0119 | Regression_pt: 0.1726 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0681 | Classification_binary: 3.2074 | Regression_cos_phi: 0.5402 | Regression_energy: 1.8711 | Regression_eta: 0.5841 | Regression_pt: 2.0695 | Regression_sin_phi: 0.4669 | LR: 9.72e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6146s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2986 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.4100 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3496.5 | uss_MB: 2718.7 | child_rss_MB: 4027.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:54:11,102] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:52,008] INFO : Step 5700/50000 rank0 | Train Loss: 9.9083 | Train Loss Unweighted: 3.5277 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.1307 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1762 | Regression_eta: 0.0141 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0706 | Classification_binary: 3.1307 | Regression_cos_phi: 0.5598 | Regression_energy: 1.9307 | Regression_eta: 0.6921 | Regression_pt: 2.0084 | Regression_sin_phi: 0.5158 | LR: 9.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4992s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2413 | max_optimizer_step: 0.1105 | max_tensorboard_logging: 0.3904 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3333.2 | uss_MB: 2555.9 | child_rss_MB: 4023.4 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:54:52,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:33,677] INFO : Step 5800/50000 rank0 | Train Loss: 10.3369 | Train Loss Unweighted: 3.5680 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1429 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1863 | Regression_eta: 0.0133 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1240 | Classification_binary: 3.1429 | Regression_cos_phi: 0.6385 | Regression_energy: 2.0416 | Regression_eta: 0.6559 | Regression_pt: 2.2035 | Regression_sin_phi: 0.5306 | LR: 9.70e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3583s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1040 | max_loss: 0.0032 | max_backward: 0.2029 | max_optimizer_step: 0.1078 | max_tensorboard_logging: 0.2815 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3333.2 | uss_MB: 2556.0 | child_rss_MB: 4026.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:55:33,893] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:14,468] INFO : Step 5900/50000 rank0 | Train Loss: 10.3648 | Train Loss Unweighted: 3.5715 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.1620 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1819 | Regression_eta: 0.0149 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1612 | Classification_binary: 3.1620 | Regression_cos_phi: 0.7267 | Regression_energy: 1.9936 | Regression_eta: 0.7315 | Regression_pt: 2.0176 | Regression_sin_phi: 0.5722 | LR: 9.69e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6689s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1038 | max_loss: 0.0029 | max_backward: 0.3941 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3932 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3333.2 | uss_MB: 2558.3 | child_rss_MB: 4026.6 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:56:14,680] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:55,836] INFO : Step 6000/50000 rank0 | Train Loss: 9.9408 | Train Loss Unweighted: 3.6323 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.2402 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1720 | Regression_eta: 0.0106 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1870 | Classification_binary: 3.2402 | Regression_cos_phi: 0.5826 | Regression_energy: 1.8846 | Regression_eta: 0.5224 | Regression_pt: 2.0087 | Regression_sin_phi: 0.5153 | LR: 9.68e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7770s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.2010 | max_loss: 0.0029 | max_backward: 0.2866 | max_optimizer_step: 0.1666 | max_tensorboard_logging: 0.3887 | max_tensorboard_flush: 0.0122 | Memory: rss_MB: 3333.2 | uss_MB: 2558.6 | child_rss_MB: 4024.6 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:56:56,050] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:56,069] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:56:56,242] INFO : Running validation on rank0 +rank0 [2026-10-28 11:56:56,242] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:56:56,242] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:56:57,477] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:56:57,621] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:56:57,752] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:56:57,826] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:56:57,895] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:56:57,963] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:56:58,032] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:56:58,101] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:56:58,207] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:56:58,336] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:56:58,467] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:56:58,538] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:56:58,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:56:58,676] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:56:58,744] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:56:58,813] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:56:58,922] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:56:58,930] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:56:58,933] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:56:58,935] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:56:58,936] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:56:58,939] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:56:58,940] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:56:58,943] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:56:58,946] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:56:58,949] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:56:58,951] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:56:58,954] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:56:58,955] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:56:58,958] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:56:58,961] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:56:58,962] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:57:06,514] INFO : VALIDATION | Step=6000/50000 | Train Loss=9.9408 | Valid Loss=10.1249 | Train Loss Unweighted=3.6323 | Valid Loss Unweighted=3.5942 | Stale=0 | ETA=326.7m +rank0 [2026-10-28 11:57:06,522] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:57:06,582] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:57:06,585] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 11:57:06,733] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:57:08,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:57:09,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:57:09,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:57:10,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:57:10,582] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:57:10,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:57:10,641] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:57:10,644] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:57:10,723] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:57:12,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:57:13,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:57:13,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:57:14,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:57:14,719] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:57:14,722] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:57:14,786] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 11:57:14,789] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 11:57:14,941] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:57:16,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:57:17,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:57:18,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:57:18,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:57:18,865] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:57:18,873] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:57:19,135] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-10-28 11:57:19,541] INFO : Plotted number of elements +rank0 [2026-10-28 11:57:21,426] INFO : Plotted jets +rank0 [2026-10-28 11:57:22,505] INFO : Plotted jet ratio +rank0 [2026-10-28 11:57:22,539] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:57:22,770] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 11:57:23,157] INFO : Plotted number of elements +rank0 [2026-10-28 11:57:24,543] INFO : Plotted jets +rank0 [2026-10-28 11:57:25,647] INFO : Plotted jet ratio +rank0 [2026-10-28 11:57:25,680] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:57:25,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_6000/cld_edm_qq_pf +rank0 [2026-10-28 11:57:26,249] INFO : Plotted number of elements +rank0 [2026-10-28 11:57:27,619] INFO : Plotted jets +rank0 [2026-10-28 11:57:28,660] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:09,510] INFO : Step 6100/50000 rank0 | Train Loss: 9.9779 | Train Loss Unweighted: 3.5781 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.1667 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1824 | Regression_eta: 0.0120 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0901 | Classification_binary: 3.1667 | Regression_cos_phi: 0.5138 | Regression_energy: 1.9985 | Regression_eta: 0.5885 | Regression_pt: 2.1424 | Regression_sin_phi: 0.4779 | LR: 9.67e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.3098 | max_optimizer_step: 0.1154 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3555.8 | uss_MB: 2778.0 | child_rss_MB: 4017.9 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:58:09,721] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:50,616] INFO : Step 6200/50000 rank0 | Train Loss: 10.9314 | Train Loss Unweighted: 3.5342 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.1056 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1825 | Regression_eta: 0.0249 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1169 | Classification_binary: 3.1056 | Regression_cos_phi: 0.7573 | Regression_energy: 1.9996 | Regression_eta: 1.2265 | Regression_pt: 2.1185 | Regression_sin_phi: 0.6068 | LR: 9.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3841s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1115 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1099 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3555.8 | uss_MB: 2778.1 | child_rss_MB: 4016.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:58:50,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:32,103] INFO : Step 6300/50000 rank0 | Train Loss: 10.4803 | Train Loss Unweighted: 3.5221 | Train Loss Components: Classification: 0.0214 | Classification_binary: 3.1039 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1827 | Regression_eta: 0.0215 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9946 | Classification_binary: 3.1039 | Regression_cos_phi: 0.6916 | Regression_energy: 2.0019 | Regression_eta: 1.0579 | Regression_pt: 2.0885 | Regression_sin_phi: 0.5418 | LR: 9.65e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5643s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0034 | max_backward: 0.2775 | max_optimizer_step: 0.1132 | max_tensorboard_logging: 0.3927 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3555.8 | uss_MB: 2778.1 | child_rss_MB: 4021.3 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:59:32,318] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:13,325] INFO : Step 6400/50000 rank0 | Train Loss: 10.0738 | Train Loss Unweighted: 3.6327 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.2357 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1738 | Regression_eta: 0.0135 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0703 | Classification_binary: 3.2357 | Regression_cos_phi: 0.6410 | Regression_energy: 1.9049 | Regression_eta: 0.6655 | Regression_pt: 2.0274 | Regression_sin_phi: 0.5292 | LR: 9.64e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.5297s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0796 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.2937 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3555.8 | uss_MB: 2778.2 | child_rss_MB: 4018.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:00:13,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:54,088] INFO : Step 6500/50000 rank0 | Train Loss: 9.7882 | Train Loss Unweighted: 3.5004 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1178 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1683 | Regression_eta: 0.0164 | Regression_pt: 0.1589 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1220 | Classification_binary: 3.1178 | Regression_cos_phi: 0.5354 | Regression_energy: 1.8437 | Regression_eta: 0.8065 | Regression_pt: 1.9060 | Regression_sin_phi: 0.4568 | LR: 9.63e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3555.8 | uss_MB: 2778.2 | child_rss_MB: 4018.8 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:00:54,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:34,847] INFO : Step 6600/50000 rank0 | Train Loss: 10.0602 | Train Loss Unweighted: 3.5393 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.1406 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1751 | Regression_eta: 0.0165 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9895 | Classification_binary: 3.1406 | Regression_cos_phi: 0.5887 | Regression_energy: 1.9188 | Regression_eta: 0.8112 | Regression_pt: 2.0131 | Regression_sin_phi: 0.5983 | LR: 9.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6091s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3111 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.3911 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3555.8 | uss_MB: 2778.2 | child_rss_MB: 4018.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:01:35,058] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:16,187] INFO : Step 6700/50000 rank0 | Train Loss: 9.0442 | Train Loss Unweighted: 3.4562 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.0980 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1589 | Regression_eta: 0.0109 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9887 | Classification_binary: 3.0980 | Regression_cos_phi: 0.4421 | Regression_energy: 1.7409 | Regression_eta: 0.5382 | Regression_pt: 1.8558 | Regression_sin_phi: 0.3805 | LR: 9.61e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.4658s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.3114 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3407.6 | uss_MB: 2630.9 | child_rss_MB: 4019.9 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:02:16,400] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:56,654] INFO : Step 6800/50000 rank0 | Train Loss: 9.4863 | Train Loss Unweighted: 3.4043 | Train Loss Components: Classification: 0.0213 | Classification_binary: 3.0100 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1756 | Regression_eta: 0.0109 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9892 | Classification_binary: 3.0100 | Regression_cos_phi: 0.4640 | Regression_energy: 1.9240 | Regression_eta: 0.5348 | Regression_pt: 2.0621 | Regression_sin_phi: 0.5023 | LR: 9.60e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5202s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0811 | max_backward: 0.2025 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3832 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3407.6 | uss_MB: 2631.2 | child_rss_MB: 4021.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:02:56,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:37,543] INFO : Step 6900/50000 rank0 | Train Loss: 9.7951 | Train Loss Unweighted: 3.5275 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.1450 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1671 | Regression_eta: 0.0132 | Regression_pt: 0.1621 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0575 | Classification_binary: 3.1450 | Regression_cos_phi: 0.6984 | Regression_energy: 1.8307 | Regression_eta: 0.6476 | Regression_pt: 1.9435 | Regression_sin_phi: 0.4724 | LR: 9.58e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6345s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1042 | max_loss: 0.0030 | max_backward: 0.3796 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3407.6 | uss_MB: 2633.3 | child_rss_MB: 4023.1 | allocated_MB: 295.1 | reserved_MB: 3650.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:03:37,757] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:18,528] INFO : Step 7000/50000 rank0 | Train Loss: 10.0575 | Train Loss Unweighted: 3.5061 | Train Loss Components: Classification: 0.0212 | Classification_binary: 3.0864 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1842 | Regression_eta: 0.0107 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9834 | Classification_binary: 3.0864 | Regression_cos_phi: 0.6562 | Regression_energy: 2.0182 | Regression_eta: 0.5261 | Regression_pt: 2.2219 | Regression_sin_phi: 0.5653 | LR: 9.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4492s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1851 | max_loss: 0.0032 | max_backward: 0.2036 | max_optimizer_step: 0.1097 | max_tensorboard_logging: 0.2879 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3407.6 | uss_MB: 2631.2 | child_rss_MB: 4023.0 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:04:18,741] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:18,760] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 12:04:18,936] INFO : Running validation on rank0 +rank0 [2026-10-28 12:04:18,937] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:04:18,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:04:20,259] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 12:04:20,404] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:04:20,534] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:04:20,608] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:04:20,677] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:04:20,745] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:04:20,815] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:04:20,883] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:04:20,990] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:04:21,120] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:04:21,251] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:04:21,321] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:04:21,390] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:04:21,457] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:04:21,526] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:04:21,596] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:04:21,706] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:04:21,712] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:04:21,716] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:04:21,719] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:04:21,721] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:04:21,722] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:04:21,725] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:04:21,728] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:04:21,732] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:04:21,734] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:04:21,735] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:04:21,738] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:04:21,741] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:04:21,742] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:04:21,745] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:04:21,748] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:04:29,344] INFO : VALIDATION | Step=7000/50000 | Train Loss=10.0575 | Valid Loss=9.6633 | Train Loss Unweighted=3.5061 | Valid Loss Unweighted=3.4446 | Stale=0 | ETA=319.0m +rank0 [2026-10-28 12:04:29,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:29,412] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:04:29,416] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:29,565] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:31,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:32,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:32,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:33,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:33,550] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:04:33,553] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:33,609] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:04:33,612] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:33,698] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:35,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:36,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:36,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:37,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:37,553] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:04:37,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:04:37,618] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:04:37,621] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:04:37,708] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:04:39,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:04:40,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:04:40,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:04:41,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:04:41,642] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:04:41,649] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:41,909] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:04:42,312] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:44,204] INFO : Plotted jets +rank0 [2026-10-28 12:04:45,284] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:45,318] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:45,549] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:04:45,939] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:47,325] INFO : Plotted jets +rank0 [2026-10-28 12:04:48,432] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:48,465] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:48,674] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_7000/cld_edm_qq_pf +rank0 [2026-10-28 12:04:49,037] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:50,408] INFO : Plotted jets +rank0 [2026-10-28 12:04:51,445] INFO : Plotted jet ratio +rank0 [2026-10-28 12:05:33,252] INFO : Step 7100/50000 rank0 | Train Loss: 10.1219 | Train Loss Unweighted: 3.4971 | Train Loss Components: Classification: 0.0209 | Classification_binary: 3.0809 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1854 | Regression_eta: 0.0177 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9709 | Classification_binary: 3.0809 | Regression_cos_phi: 0.5381 | Regression_energy: 2.0313 | Regression_eta: 0.8706 | Regression_pt: 2.1143 | Regression_sin_phi: 0.5158 | LR: 9.56e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3042 | max_optimizer_step: 0.1156 | max_tensorboard_logging: 0.4048 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4025.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:05:33,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:14,934] INFO : Step 7200/50000 rank0 | Train Loss: 8.9809 | Train Loss Unweighted: 3.4482 | Train Loss Components: Classification: 0.0208 | Classification_binary: 3.0948 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1561 | Regression_eta: 0.0098 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9630 | Classification_binary: 3.0948 | Regression_cos_phi: 0.4916 | Regression_energy: 1.7109 | Regression_eta: 0.4805 | Regression_pt: 1.8387 | Regression_sin_phi: 0.4014 | LR: 9.55e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4672s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1830 | max_loss: 0.0031 | max_backward: 0.2023 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4025.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:06:15,147] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:55,739] INFO : Step 7300/50000 rank0 | Train Loss: 9.7861 | Train Loss Unweighted: 3.4673 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.0670 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1746 | Regression_eta: 0.0122 | Regression_pt: 0.1751 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0302 | Classification_binary: 3.0670 | Regression_cos_phi: 0.5820 | Regression_energy: 1.9136 | Regression_eta: 0.6009 | Regression_pt: 2.0999 | Regression_sin_phi: 0.4924 | LR: 9.53e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5332s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0030 | max_backward: 0.2624 | max_optimizer_step: 0.1128 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4020.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:06:55,955] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:36,885] INFO : Step 7400/50000 rank0 | Train Loss: 10.5199 | Train Loss Unweighted: 3.5094 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0572 | Regression_cos_phi: 0.0077 | Regression_energy: 0.2018 | Regression_eta: 0.0181 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0190 | Classification_binary: 3.0572 | Regression_cos_phi: 0.5445 | Regression_energy: 2.2113 | Regression_eta: 0.8896 | Regression_pt: 2.3426 | Regression_sin_phi: 0.4557 | LR: 9.52e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5008s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0458 | max_forward: 0.0772 | max_loss: 0.0031 | max_backward: 0.2021 | max_optimizer_step: 0.1826 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4020.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:07:37,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:17,453] INFO : Step 7500/50000 rank0 | Train Loss: 10.3226 | Train Loss Unweighted: 3.5274 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.1113 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1820 | Regression_eta: 0.0163 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.1562 | Classification_binary: 3.1113 | Regression_cos_phi: 0.6400 | Regression_energy: 1.9945 | Regression_eta: 0.8017 | Regression_pt: 2.1050 | Regression_sin_phi: 0.5139 | LR: 9.51e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5527s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0990 | max_backward: 0.2025 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4020.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:08:17,664] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:59,104] INFO : Step 7600/50000 rank0 | Train Loss: 10.8716 | Train Loss Unweighted: 3.6769 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.2321 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1977 | Regression_eta: 0.0202 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0250 | Classification_binary: 3.2321 | Regression_cos_phi: 0.6214 | Regression_energy: 2.1667 | Regression_eta: 0.9914 | Regression_pt: 2.2373 | Regression_sin_phi: 0.5978 | LR: 9.50e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5349s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0435 | max_backward: 0.2020 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3635.4 | uss_MB: 2858.8 | child_rss_MB: 4021.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:08:59,316] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:40,620] INFO : Step 7700/50000 rank0 | Train Loss: 9.6674 | Train Loss Unweighted: 3.3850 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.9820 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1789 | Regression_eta: 0.0112 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0125 | Classification_binary: 2.9820 | Regression_cos_phi: 0.5771 | Regression_energy: 1.9599 | Regression_eta: 0.5493 | Regression_pt: 2.1027 | Regression_sin_phi: 0.4839 | LR: 9.48e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5488s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0031 | max_backward: 0.3094 | max_optimizer_step: 0.1115 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3484.3 | uss_MB: 2706.0 | child_rss_MB: 4024.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:09:40,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:22,352] INFO : Step 7800/50000 rank0 | Train Loss: 9.8552 | Train Loss Unweighted: 3.5135 | Train Loss Components: Classification: 0.0210 | Classification_binary: 3.1000 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1842 | Regression_eta: 0.0118 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9735 | Classification_binary: 3.1000 | Regression_cos_phi: 0.5362 | Regression_energy: 2.0181 | Regression_eta: 0.5780 | Regression_pt: 2.1760 | Regression_sin_phi: 0.4735 | LR: 9.47e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5411s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0927 | max_backward: 0.2023 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3485.3 | uss_MB: 2706.3 | child_rss_MB: 4024.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:10:22,564] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:03,199] INFO : Step 7900/50000 rank0 | Train Loss: 9.6907 | Train Loss Unweighted: 3.4372 | Train Loss Components: Classification: 0.0212 | Classification_binary: 3.0498 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1738 | Regression_eta: 0.0162 | Regression_pt: 0.1607 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9858 | Classification_binary: 3.0498 | Regression_cos_phi: 0.4973 | Regression_energy: 1.9047 | Regression_eta: 0.7964 | Regression_pt: 1.9267 | Regression_sin_phi: 0.5300 | LR: 9.46e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4987s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0687 | max_backward: 0.2022 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3485.3 | uss_MB: 2706.4 | child_rss_MB: 4021.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:11:03,414] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:44,369] INFO : Step 8000/50000 rank0 | Train Loss: 9.3785 | Train Loss Unweighted: 3.3889 | Train Loss Components: Classification: 0.0226 | Classification_binary: 3.0202 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1602 | Regression_eta: 0.0107 | Regression_pt: 0.1584 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0503 | Classification_binary: 3.0202 | Regression_cos_phi: 0.6913 | Regression_energy: 1.7558 | Regression_eta: 0.5244 | Regression_pt: 1.8994 | Regression_sin_phi: 0.4370 | LR: 9.44e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.4318s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3080 | max_optimizer_step: 0.1100 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3485.3 | uss_MB: 2706.5 | child_rss_MB: 4044.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:11:44,582] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:44,600] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 12:11:44,773] INFO : Running validation on rank0 +rank0 [2026-10-28 12:11:44,773] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:11:44,773] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:11:46,024] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 12:11:46,167] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:11:46,299] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:11:46,373] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:11:46,442] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:11:46,513] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:11:46,582] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:11:46,651] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:11:46,760] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:11:46,890] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:11:47,020] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:11:47,091] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:11:47,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:11:47,228] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:11:47,297] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:11:47,366] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:11:47,475] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:11:47,482] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:11:47,485] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:11:47,488] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:11:47,489] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:11:47,492] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:11:47,495] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:11:47,498] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:11:47,500] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:11:47,501] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:11:47,504] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:11:47,506] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:11:47,507] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:11:47,510] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:11:47,512] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:11:47,514] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:11:55,089] INFO : VALIDATION | Step=8000/50000 | Train Loss=9.3785 | Valid Loss=9.5710 | Train Loss Unweighted=3.3889 | Valid Loss Unweighted=3.4009 | Stale=0 | ETA=311.7m +rank0 [2026-10-28 12:11:55,098] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:55,158] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:11:55,162] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:11:55,310] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:57,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:58,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:58,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:59,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:59,497] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:11:59,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:59,556] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:11:59,559] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:11:59,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:01,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:02,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:02,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:03,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:03,595] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:12:03,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:12:03,653] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:12:03,656] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:12:03,812] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:12:05,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:12:06,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:12:06,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:12:07,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:12:07,697] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:12:07,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:07,961] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:12:08,364] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:10,277] INFO : Plotted jets +rank0 [2026-10-28 12:12:11,359] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:11,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:11,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:12,018] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:13,410] INFO : Plotted jets +rank0 [2026-10-28 12:12:14,508] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:14,540] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 12:12:14,747] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_8000/cld_edm_qq_pf +rank0 [2026-10-28 12:12:15,108] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:16,479] INFO : Plotted jets +rank0 [2026-10-28 12:12:17,520] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:58,909] INFO : Step 8100/50000 rank0 | Train Loss: 9.3781 | Train Loss Unweighted: 3.2862 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9150 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1632 | Regression_eta: 0.0146 | Regression_pt: 0.1554 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0052 | Classification_binary: 2.9150 | Regression_cos_phi: 0.5202 | Regression_energy: 1.7879 | Regression_eta: 0.7199 | Regression_pt: 1.8633 | Regression_sin_phi: 0.5665 | LR: 9.43e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1093 | max_forward: 0.1039 | max_loss: 0.0031 | max_backward: 0.3937 | max_optimizer_step: 0.1145 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3684.9 | uss_MB: 2912.4 | child_rss_MB: 4045.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:12:59,123] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:39,905] INFO : Step 8200/50000 rank0 | Train Loss: 9.3862 | Train Loss Unweighted: 3.4096 | Train Loss Components: Classification: 0.0196 | Classification_binary: 3.0381 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1628 | Regression_eta: 0.0158 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9092 | Classification_binary: 3.0381 | Regression_cos_phi: 0.4944 | Regression_energy: 1.7833 | Regression_eta: 0.7755 | Regression_pt: 1.9044 | Regression_sin_phi: 0.4812 | LR: 9.42e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5903s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0033 | max_backward: 0.3101 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3684.9 | uss_MB: 2910.4 | child_rss_MB: 4017.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:13:40,119] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:20,833] INFO : Step 8300/50000 rank0 | Train Loss: 9.1849 | Train Loss Unweighted: 3.2781 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.9016 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1674 | Regression_eta: 0.0107 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9062 | Classification_binary: 2.9016 | Regression_cos_phi: 0.6362 | Regression_energy: 1.8344 | Regression_eta: 0.5255 | Regression_pt: 1.9566 | Regression_sin_phi: 0.4243 | LR: 9.40e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.2266 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3684.9 | uss_MB: 2910.5 | child_rss_MB: 4019.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:14:21,047] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:02,467] INFO : Step 8400/50000 rank0 | Train Loss: 9.2624 | Train Loss Unweighted: 3.4558 | Train Loss Components: Classification: 0.0202 | Classification_binary: 3.0929 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1600 | Regression_eta: 0.0125 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9395 | Classification_binary: 3.0929 | Regression_cos_phi: 0.5417 | Regression_energy: 1.7526 | Regression_eta: 0.6138 | Regression_pt: 1.8622 | Regression_sin_phi: 0.4596 | LR: 9.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.2927 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3964 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3684.9 | uss_MB: 2910.5 | child_rss_MB: 4019.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:15:02,683] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:43,243] INFO : Step 8500/50000 rank0 | Train Loss: 9.6020 | Train Loss Unweighted: 3.4346 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0506 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1685 | Regression_eta: 0.0116 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0574 | Classification_binary: 3.0506 | Regression_cos_phi: 0.5681 | Regression_energy: 1.8465 | Regression_eta: 0.5686 | Regression_pt: 1.9739 | Regression_sin_phi: 0.5368 | LR: 9.37e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6650s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3105 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.4194 | max_tensorboard_flush: 0.0134 | Memory: rss_MB: 3685.9 | uss_MB: 2910.5 | child_rss_MB: 4018.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:15:43,458] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:24,238] INFO : Step 8600/50000 rank0 | Train Loss: 9.2516 | Train Loss Unweighted: 3.3888 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.0301 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1559 | Regression_eta: 0.0124 | Regression_pt: 0.1527 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0053 | Classification_binary: 3.0301 | Regression_cos_phi: 0.5530 | Regression_energy: 1.7083 | Regression_eta: 0.6089 | Regression_pt: 1.8309 | Regression_sin_phi: 0.5150 | LR: 9.36e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0992 | max_loss: 0.0032 | max_backward: 0.2025 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3156 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3685.9 | uss_MB: 2910.5 | child_rss_MB: 4020.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:16:24,457] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:05,328] INFO : Step 8700/50000 rank0 | Train Loss: 9.8021 | Train Loss Unweighted: 3.3609 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9745 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1702 | Regression_eta: 0.0173 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0138 | Classification_binary: 2.9745 | Regression_cos_phi: 0.6777 | Regression_energy: 1.8647 | Regression_eta: 0.8485 | Regression_pt: 1.9118 | Regression_sin_phi: 0.5110 | LR: 9.34e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4798s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0031 | max_backward: 0.3092 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2976 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3554.2 | uss_MB: 2778.7 | child_rss_MB: 4045.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:17:05,542] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:46,398] INFO : Step 8800/50000 rank0 | Train Loss: 9.1497 | Train Loss Unweighted: 3.4013 | Train Loss Components: Classification: 0.0228 | Classification_binary: 3.0462 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1557 | Regression_eta: 0.0111 | Regression_pt: 0.1508 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0584 | Classification_binary: 3.0462 | Regression_cos_phi: 0.5452 | Regression_energy: 1.7061 | Regression_eta: 0.5459 | Regression_pt: 1.8081 | Regression_sin_phi: 0.4399 | LR: 9.33e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5457s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0033 | max_backward: 0.3094 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3554.2 | uss_MB: 2778.6 | child_rss_MB: 4013.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:17:46,612] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:26,962] INFO : Step 8900/50000 rank0 | Train Loss: 9.3941 | Train Loss Unweighted: 3.3843 | Train Loss Components: Classification: 0.0202 | Classification_binary: 3.0078 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1658 | Regression_eta: 0.0119 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9387 | Classification_binary: 3.0078 | Regression_cos_phi: 0.5930 | Regression_energy: 1.8165 | Regression_eta: 0.5848 | Regression_pt: 1.9449 | Regression_sin_phi: 0.5084 | LR: 9.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4546s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0226 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.2023 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3554.2 | uss_MB: 2778.4 | child_rss_MB: 4017.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 12:18:27,193] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:07,606] INFO : Step 9000/50000 rank0 | Train Loss: 8.6657 | Train Loss Unweighted: 3.1986 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.8525 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1518 | Regression_eta: 0.0085 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0267 | Classification_binary: 2.8525 | Regression_cos_phi: 0.5075 | Regression_energy: 1.6629 | Regression_eta: 0.4169 | Regression_pt: 1.8007 | Regression_sin_phi: 0.3985 | LR: 9.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5433s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0833 | max_backward: 0.2028 | max_optimizer_step: 0.1103 | max_tensorboard_logging: 0.4005 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3554.2 | uss_MB: 2778.5 | child_rss_MB: 4017.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:19:07,820] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:07,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 12:19:08,029] INFO : Running validation on rank0 +rank0 [2026-10-28 12:19:08,029] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:19:08,029] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:19:09,363] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 12:19:09,507] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:19:09,638] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:19:09,712] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:19:09,781] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:19:09,849] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:19:09,919] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:19:09,988] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:19:10,098] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:19:10,229] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:19:10,359] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:19:10,430] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:19:10,498] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:19:10,567] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:19:10,635] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:19:10,704] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:19:10,807] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:19:10,814] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:19:10,817] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:19:10,820] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:19:10,821] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:19:10,824] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:19:10,826] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:19:10,828] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:19:10,831] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:19:10,834] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:19:10,836] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:19:10,838] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:19:10,842] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:19:10,844] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:19:10,846] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:19:10,849] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:19:18,318] INFO : VALIDATION | Step=9000/50000 | Train Loss=8.6657 | Valid Loss=9.5121 | Train Loss Unweighted=3.1986 | Valid Loss Unweighted=3.3604 | Stale=0 | ETA=304.1m +rank0 [2026-10-28 12:19:18,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:18,415] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:19:18,418] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:19:18,662] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:20,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:21,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:22,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:22,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:22,730] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:19:22,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:22,788] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:19:22,791] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:22,876] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:24,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:25,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:25,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:26,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:26,757] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:19:26,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:26,836] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:19:26,839] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:19:26,915] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:28,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:29,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:30,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:30,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:30,871] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:19:30,877] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,134] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,539] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:33,463] INFO : Plotted jets +rank0 [2026-10-28 12:19:34,544] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:34,576] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:34,809] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:35,196] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:36,585] INFO : Plotted jets +rank0 [2026-10-28 12:19:37,689] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:37,722] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 12:19:37,930] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_9000/cld_edm_qq_pf +rank0 [2026-10-28 12:19:38,294] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:39,667] INFO : Plotted jets +rank0 [2026-10-28 12:19:40,711] INFO : Plotted jet ratio +rank0 [2026-10-28 12:20:21,554] INFO : Step 9100/50000 rank0 | Train Loss: 9.2302 | Train Loss Unweighted: 3.3404 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.9630 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1657 | Regression_eta: 0.0112 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9607 | Classification_binary: 2.9630 | Regression_cos_phi: 0.5087 | Regression_energy: 1.8154 | Regression_eta: 0.5522 | Regression_pt: 1.9848 | Regression_sin_phi: 0.4455 | LR: 9.28e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5622s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1857 | max_loss: 0.0033 | max_backward: 0.2023 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3364 | max_tensorboard_flush: 0.0151 | Memory: rss_MB: 3625.5 | uss_MB: 2850.6 | child_rss_MB: 4016.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:20:21,776] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:02,602] INFO : Step 9200/50000 rank0 | Train Loss: 10.4475 | Train Loss Unweighted: 3.4601 | Train Loss Components: Classification: 0.0216 | Classification_binary: 3.0262 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1927 | Regression_eta: 0.0201 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0026 | Classification_binary: 3.0262 | Regression_cos_phi: 0.6756 | Regression_energy: 2.1110 | Regression_eta: 0.9889 | Regression_pt: 2.1930 | Regression_sin_phi: 0.4501 | LR: 9.27e-04 | DataLoad Time: 0.0034s | Model Forward Time: 1.5510s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0618 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2023 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 0.3111 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3627.5 | uss_MB: 2852.1 | child_rss_MB: 4016.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:21:02,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:44,018] INFO : Step 9300/50000 rank0 | Train Loss: 9.1003 | Train Loss Unweighted: 3.2582 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.8962 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1585 | Regression_eta: 0.0151 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9310 | Classification_binary: 2.8962 | Regression_cos_phi: 0.4409 | Regression_energy: 1.7362 | Regression_eta: 0.7405 | Regression_pt: 1.8489 | Regression_sin_phi: 0.5067 | LR: 9.25e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5364s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1816 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.3189 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3627.5 | uss_MB: 2852.3 | child_rss_MB: 4016.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:21:44,234] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:25,642] INFO : Step 9400/50000 rank0 | Train Loss: 10.2354 | Train Loss Unweighted: 3.5125 | Train Loss Components: Classification: 0.0196 | Classification_binary: 3.1231 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1696 | Regression_eta: 0.0239 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9089 | Classification_binary: 3.1231 | Regression_cos_phi: 0.6816 | Regression_energy: 1.8583 | Regression_eta: 1.1761 | Regression_pt: 1.8824 | Regression_sin_phi: 0.6050 | LR: 9.24e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5919s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0636 | max_backward: 0.2023 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.3878 | max_tensorboard_flush: 0.0128 | Memory: rss_MB: 3627.5 | uss_MB: 2852.4 | child_rss_MB: 4015.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:22:25,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:06,786] INFO : Step 9500/50000 rank0 | Train Loss: 8.5242 | Train Loss Unweighted: 3.1861 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.8469 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1501 | Regression_eta: 0.0084 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9586 | Classification_binary: 2.8469 | Regression_cos_phi: 0.4932 | Regression_energy: 1.6448 | Regression_eta: 0.4139 | Regression_pt: 1.7573 | Regression_sin_phi: 0.4095 | LR: 9.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5502s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0409 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.2026 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.3144 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 3627.5 | uss_MB: 2852.4 | child_rss_MB: 4019.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:23:07,000] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:48,785] INFO : Step 9600/50000 rank0 | Train Loss: 8.7320 | Train Loss Unweighted: 3.2264 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.8762 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1556 | Regression_eta: 0.0106 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9608 | Classification_binary: 2.8762 | Regression_cos_phi: 0.4732 | Regression_energy: 1.7046 | Regression_eta: 0.5227 | Regression_pt: 1.8026 | Regression_sin_phi: 0.3919 | LR: 9.21e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5900s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.3028 | max_optimizer_step: 0.1171 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3627.5 | uss_MB: 2852.4 | child_rss_MB: 4018.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:23:48,999] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:30,492] INFO : Step 9700/50000 rank0 | Train Loss: 8.9081 | Train Loss Unweighted: 3.2550 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.8900 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1618 | Regression_eta: 0.0105 | Regression_pt: 0.1589 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9888 | Classification_binary: 2.8900 | Regression_cos_phi: 0.4276 | Regression_energy: 1.7729 | Regression_eta: 0.5159 | Regression_pt: 1.9051 | Regression_sin_phi: 0.4077 | LR: 9.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5805s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0032 | max_backward: 0.3092 | max_optimizer_step: 0.1132 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3627.0 | uss_MB: 2850.5 | child_rss_MB: 4015.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:24:30,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:12,200] INFO : Step 9800/50000 rank0 | Train Loss: 8.9568 | Train Loss Unweighted: 3.2080 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8562 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1559 | Regression_eta: 0.0141 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9433 | Classification_binary: 2.8562 | Regression_cos_phi: 0.5647 | Regression_energy: 1.7085 | Regression_eta: 0.6925 | Regression_pt: 1.7583 | Regression_sin_phi: 0.4333 | LR: 9.17e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4842s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0286 | max_backward: 0.2027 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.3781 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3627.0 | uss_MB: 2852.0 | child_rss_MB: 4017.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:25:12,412] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:52,681] INFO : Step 9900/50000 rank0 | Train Loss: 9.9009 | Train Loss Unweighted: 3.4068 | Train Loss Components: Classification: 0.0209 | Classification_binary: 3.0022 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1780 | Regression_eta: 0.0125 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9681 | Classification_binary: 3.0022 | Regression_cos_phi: 0.5704 | Regression_energy: 1.9502 | Regression_eta: 0.6137 | Regression_pt: 2.0861 | Regression_sin_phi: 0.7102 | LR: 9.16e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6715s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2127 | max_loss: 0.0034 | max_backward: 0.2861 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.2856 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3627.0 | uss_MB: 2854.6 | child_rss_MB: 4014.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:25:52,912] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:33,223] INFO : Step 10000/50000 rank0 | Train Loss: 9.4590 | Train Loss Unweighted: 3.3266 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9277 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1790 | Regression_eta: 0.0100 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9832 | Classification_binary: 2.9277 | Regression_cos_phi: 0.5710 | Regression_energy: 1.9618 | Regression_eta: 0.4929 | Regression_pt: 2.0818 | Regression_sin_phi: 0.4406 | LR: 9.14e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6978s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3937 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3909 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3627.0 | uss_MB: 2854.7 | child_rss_MB: 4013.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:26:33,437] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:33,457] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 12:26:33,733] INFO : Running validation on rank0 +rank0 [2026-10-28 12:26:33,733] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:26:33,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:26:35,039] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 12:26:35,185] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:26:35,316] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:26:35,391] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:26:35,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:26:35,528] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:26:35,598] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:26:35,667] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:26:35,775] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:26:35,907] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:26:36,038] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:26:36,110] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:26:36,179] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:26:36,247] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:26:36,316] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:26:36,385] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:26:36,488] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:26:36,494] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:26:36,497] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:26:36,500] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:26:36,501] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:26:36,504] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:26:36,505] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:26:36,508] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:26:36,510] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:26:36,514] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:26:36,515] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:26:36,517] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:26:36,518] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:26:36,520] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:26:36,521] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:26:36,524] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:26:43,996] INFO : VALIDATION | Step=10000/50000 | Train Loss=9.4590 | Valid Loss=9.1721 | Train Loss Unweighted=3.3266 | Valid Loss Unweighted=3.2697 | Stale=0 | ETA=296.7m +rank0 [2026-10-28 12:26:44,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:44,076] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:26:44,080] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:26:44,285] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:46,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:46,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:47,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:48,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:48,401] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:26:48,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:48,472] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:26:48,475] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:48,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:50,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:51,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:51,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:52,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:52,477] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:26:52,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:52,568] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:26:52,571] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:26:52,637] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:54,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:55,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:55,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:56,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:56,581] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:26:56,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:26:56,843] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:26:57,247] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:59,151] INFO : Plotted jets +rank0 [2026-10-28 12:27:00,226] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:00,259] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:00,491] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:27:00,879] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:02,259] INFO : Plotted jets +rank0 [2026-10-28 12:27:03,362] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:03,395] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:03,602] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_10000/cld_edm_qq_pf +rank0 [2026-10-28 12:27:03,966] INFO : Plotted number of elements +rank0 [2026-10-28 12:27:05,341] INFO : Plotted jets +rank0 [2026-10-28 12:27:06,389] INFO : Plotted jet ratio +rank0 [2026-10-28 12:27:47,598] INFO : Step 10100/50000 rank0 | Train Loss: 9.3153 | Train Loss Unweighted: 3.3198 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9753 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1455 | Regression_eta: 0.0177 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0975 | Classification_binary: 2.9753 | Regression_cos_phi: 0.6233 | Regression_energy: 1.5939 | Regression_eta: 0.8685 | Regression_pt: 1.6989 | Regression_sin_phi: 0.4580 | LR: 9.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6437s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3103 | max_optimizer_step: 0.1215 | max_tensorboard_logging: 0.4122 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3699.6 | uss_MB: 2924.9 | child_rss_MB: 4008.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:27:47,812] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:29,152] INFO : Step 10200/50000 rank0 | Train Loss: 9.2086 | Train Loss Unweighted: 3.2862 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9299 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1531 | Regression_eta: 0.0158 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0409 | Classification_binary: 2.9299 | Regression_cos_phi: 0.5289 | Regression_energy: 1.6779 | Regression_eta: 0.7771 | Regression_pt: 1.8017 | Regression_sin_phi: 0.4521 | LR: 9.11e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5240s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2435 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3787 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3700.6 | uss_MB: 2926.5 | child_rss_MB: 4005.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:28:29,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:09,728] INFO : Step 10300/50000 rank0 | Train Loss: 10.6029 | Train Loss Unweighted: 3.4198 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.9795 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1933 | Regression_eta: 0.0238 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9965 | Classification_binary: 2.9795 | Regression_cos_phi: 0.5911 | Regression_energy: 2.1185 | Regression_eta: 1.1683 | Regression_pt: 2.2167 | Regression_sin_phi: 0.5323 | LR: 9.09e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5208s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1790 | max_loss: 0.0030 | max_backward: 0.2024 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2999 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3700.6 | uss_MB: 2926.6 | child_rss_MB: 4009.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:29:09,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:51,402] INFO : Step 10400/50000 rank0 | Train Loss: 8.9836 | Train Loss Unweighted: 3.2755 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.9044 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1650 | Regression_eta: 0.0087 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0021 | Classification_binary: 2.9044 | Regression_cos_phi: 0.4515 | Regression_energy: 1.8080 | Regression_eta: 0.4266 | Regression_pt: 1.9478 | Regression_sin_phi: 0.4432 | LR: 9.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.2580 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3864 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3700.6 | uss_MB: 2926.6 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:29:51,616] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:32,417] INFO : Step 10500/50000 rank0 | Train Loss: 9.2066 | Train Loss Unweighted: 3.2772 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9076 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1616 | Regression_eta: 0.0109 | Regression_pt: 0.1584 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0956 | Classification_binary: 2.9076 | Regression_cos_phi: 0.5005 | Regression_energy: 1.7706 | Regression_eta: 0.5371 | Regression_pt: 1.9000 | Regression_sin_phi: 0.4951 | LR: 9.06e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3107 | max_optimizer_step: 0.1207 | max_tensorboard_logging: 0.2773 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3700.6 | uss_MB: 2926.5 | child_rss_MB: 4011.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:30:32,630] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:13,494] INFO : Step 10600/50000 rank0 | Train Loss: 9.6617 | Train Loss Unweighted: 3.3332 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9353 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1764 | Regression_eta: 0.0145 | Regression_pt: 0.1699 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9827 | Classification_binary: 2.9353 | Regression_cos_phi: 0.5741 | Regression_energy: 1.9329 | Regression_eta: 0.7139 | Regression_pt: 2.0373 | Regression_sin_phi: 0.4854 | LR: 9.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5721s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2834 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3844 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3700.6 | uss_MB: 2926.7 | child_rss_MB: 4011.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:31:13,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:54,887] INFO : Step 10700/50000 rank0 | Train Loss: 9.7603 | Train Loss Unweighted: 3.3193 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9321 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1701 | Regression_eta: 0.0195 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9828 | Classification_binary: 2.9321 | Regression_cos_phi: 0.5325 | Regression_energy: 1.8643 | Regression_eta: 0.9591 | Regression_pt: 1.9154 | Regression_sin_phi: 0.5741 | LR: 9.02e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4991s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1811 | max_loss: 0.0031 | max_backward: 0.2023 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.2991 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3700.1 | uss_MB: 2925.0 | child_rss_MB: 4013.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:31:55,099] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:36,749] INFO : Step 10800/50000 rank0 | Train Loss: 9.2213 | Train Loss Unweighted: 3.2397 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.8608 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1670 | Regression_eta: 0.0092 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0908 | Classification_binary: 2.8608 | Regression_cos_phi: 0.5574 | Regression_energy: 1.8302 | Regression_eta: 0.4543 | Regression_pt: 1.9643 | Regression_sin_phi: 0.4634 | LR: 9.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5933s | Phase Times: max_pre_step_barrier: 0.1051 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.3061 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3701.1 | uss_MB: 2926.6 | child_rss_MB: 4034.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:32:36,964] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:18,470] INFO : Step 10900/50000 rank0 | Train Loss: 8.5178 | Train Loss Unweighted: 3.2527 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.9138 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1507 | Regression_eta: 0.0087 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9381 | Classification_binary: 2.9138 | Regression_cos_phi: 0.4330 | Regression_energy: 1.6517 | Regression_eta: 0.4277 | Regression_pt: 1.7622 | Regression_sin_phi: 0.3913 | LR: 8.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5844s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3058 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.3897 | max_tensorboard_flush: 0.0056 | Memory: rss_MB: 3701.1 | uss_MB: 2926.6 | child_rss_MB: 4016.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:33:18,686] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:59,841] INFO : Step 11000/50000 rank0 | Train Loss: 8.5435 | Train Loss Unweighted: 3.2313 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8933 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1511 | Regression_eta: 0.0099 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9237 | Classification_binary: 2.8933 | Regression_cos_phi: 0.4400 | Regression_energy: 1.6553 | Regression_eta: 0.4870 | Regression_pt: 1.7307 | Regression_sin_phi: 0.4136 | LR: 8.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5330s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0831 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2022 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3701.1 | uss_MB: 2926.7 | child_rss_MB: 4016.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:34:00,075] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:00,094] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 12:34:00,270] INFO : Running validation on rank0 +rank0 [2026-10-28 12:34:00,271] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:34:00,271] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:34:01,583] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 12:34:01,729] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:34:01,861] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:34:01,937] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:34:02,007] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:34:02,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:34:02,146] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:34:02,215] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:34:02,324] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:34:02,455] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:34:02,586] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:34:02,657] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:34:02,726] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:34:02,794] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:34:02,864] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:34:02,933] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:34:03,051] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:34:03,058] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:34:03,061] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:34:03,064] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:34:03,065] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:34:03,068] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:34:03,070] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:34:03,072] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:34:03,074] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:34:03,078] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:34:03,080] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:34:03,081] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:34:03,084] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:34:03,086] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:34:03,089] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:34:03,090] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:34:10,668] INFO : VALIDATION | Step=11000/50000 | Train Loss=8.5435 | Valid Loss=8.9201 | Train Loss Unweighted=3.2313 | Valid Loss Unweighted=3.1908 | Stale=0 | ETA=289.4m +rank0 [2026-10-28 12:34:10,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:10,735] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:34:10,739] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:10,893] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:12,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:13,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:14,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:15,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:15,103] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:34:15,106] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:15,161] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:34:15,164] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:15,262] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:17,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:17,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:18,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:19,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:19,241] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:34:19,244] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:34:19,298] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:34:19,302] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:34:19,403] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:34:21,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:34:21,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:34:22,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:34:23,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:34:23,360] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:34:23,367] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:23,621] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:34:24,026] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:25,922] INFO : Plotted jets +rank0 [2026-10-28 12:34:27,001] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:27,034] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:27,265] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:34:27,657] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:29,048] INFO : Plotted jets +rank0 [2026-10-28 12:34:30,157] INFO : Plotted jet ratio +rank0 [2026-10-28 12:34:30,190] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:30,399] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_11000/cld_edm_qq_pf +rank0 [2026-10-28 12:34:30,759] INFO : Plotted number of elements +rank0 [2026-10-28 12:34:32,142] INFO : Plotted jets +rank0 [2026-10-28 12:34:33,200] INFO : Plotted jet ratio +rank0 [2026-10-28 12:35:14,348] INFO : Step 11100/50000 rank0 | Train Loss: 8.9500 | Train Loss Unweighted: 3.0962 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7295 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1615 | Regression_eta: 0.0120 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9195 | Classification_binary: 2.7295 | Regression_cos_phi: 0.5917 | Regression_energy: 1.7695 | Regression_eta: 0.5893 | Regression_pt: 1.8917 | Regression_sin_phi: 0.4588 | LR: 8.95e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8261s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0879 | max_forward: 0.1040 | max_loss: 0.0031 | max_backward: 0.3934 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3003 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3771.7 | uss_MB: 3001.9 | child_rss_MB: 4015.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:35:14,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:56,085] INFO : Step 11200/50000 rank0 | Train Loss: 9.3948 | Train Loss Unweighted: 3.2278 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8492 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1662 | Regression_eta: 0.0192 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9224 | Classification_binary: 2.8492 | Regression_cos_phi: 0.5050 | Regression_energy: 1.8213 | Regression_eta: 0.9445 | Regression_pt: 1.9080 | Regression_sin_phi: 0.4444 | LR: 8.93e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5850s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0014 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.3096 | max_optimizer_step: 0.1150 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3773.7 | uss_MB: 3001.4 | child_rss_MB: 4014.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:35:56,300] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:36,433] INFO : Step 11300/50000 rank0 | Train Loss: 8.5869 | Train Loss Unweighted: 3.1868 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.8434 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1518 | Regression_eta: 0.0085 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9994 | Classification_binary: 2.8434 | Regression_cos_phi: 0.5066 | Regression_energy: 1.6637 | Regression_eta: 0.4194 | Regression_pt: 1.7782 | Regression_sin_phi: 0.3762 | LR: 8.91e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1028 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3773.7 | uss_MB: 3001.7 | child_rss_MB: 4014.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:36:36,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:17,850] INFO : Step 11400/50000 rank0 | Train Loss: 10.0288 | Train Loss Unweighted: 3.4148 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9918 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1905 | Regression_eta: 0.0127 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0384 | Classification_binary: 2.9918 | Regression_cos_phi: 0.6083 | Regression_energy: 2.0872 | Regression_eta: 0.6250 | Regression_pt: 2.1674 | Regression_sin_phi: 0.5107 | LR: 8.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5586s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2683 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3773.7 | uss_MB: 3001.7 | child_rss_MB: 4008.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:37:18,063] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:58,745] INFO : Step 11500/50000 rank0 | Train Loss: 9.4975 | Train Loss Unweighted: 3.3741 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.9963 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1641 | Regression_eta: 0.0149 | Regression_pt: 0.1619 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9843 | Classification_binary: 2.9963 | Regression_cos_phi: 0.5135 | Regression_energy: 1.7986 | Regression_eta: 0.7316 | Regression_pt: 1.9419 | Regression_sin_phi: 0.5313 | LR: 8.88e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.3069 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3843 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3773.7 | uss_MB: 3001.8 | child_rss_MB: 4011.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:37:58,959] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:38,960] INFO : Step 11600/50000 rank0 | Train Loss: 9.3977 | Train Loss Unweighted: 3.3901 | Train Loss Components: Classification: 0.0211 | Classification_binary: 3.0019 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1749 | Regression_eta: 0.0108 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9810 | Classification_binary: 3.0019 | Regression_cos_phi: 0.4921 | Regression_energy: 1.9162 | Regression_eta: 0.5291 | Regression_pt: 2.0010 | Regression_sin_phi: 0.4764 | LR: 8.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5539s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0913 | max_backward: 0.2026 | max_optimizer_step: 0.1125 | max_tensorboard_logging: 0.3802 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3773.7 | uss_MB: 3002.0 | child_rss_MB: 4011.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:38:39,174] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:20,497] INFO : Step 11700/50000 rank0 | Train Loss: 9.0855 | Train Loss Unweighted: 3.1975 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.8451 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1543 | Regression_eta: 0.0172 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0000 | Classification_binary: 2.8451 | Regression_cos_phi: 0.5402 | Regression_energy: 1.6902 | Regression_eta: 0.8459 | Regression_pt: 1.7381 | Regression_sin_phi: 0.4259 | LR: 8.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.9721s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0534 | max_forward: 0.4139 | max_loss: 0.1033 | max_backward: 0.2029 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3771.8 | uss_MB: 3000.0 | child_rss_MB: 4010.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:39:20,708] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:01,569] INFO : Step 11800/50000 rank0 | Train Loss: 9.1380 | Train Loss Unweighted: 3.1130 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7255 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1730 | Regression_eta: 0.0122 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8952 | Classification_binary: 2.7255 | Regression_cos_phi: 0.5840 | Regression_energy: 1.8951 | Regression_eta: 0.6005 | Regression_pt: 2.0148 | Regression_sin_phi: 0.4229 | LR: 8.82e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2084 | max_loss: 0.0031 | max_backward: 0.2861 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.2795 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3771.8 | uss_MB: 3003.2 | child_rss_MB: 4010.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:40:01,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:43,375] INFO : Step 11900/50000 rank0 | Train Loss: 9.0896 | Train Loss Unweighted: 3.1849 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.8375 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1493 | Regression_eta: 0.0172 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9511 | Classification_binary: 2.8375 | Regression_cos_phi: 0.5635 | Regression_energy: 1.6356 | Regression_eta: 0.8438 | Regression_pt: 1.7271 | Regression_sin_phi: 0.5310 | LR: 8.80e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5422s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.2970 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.3735 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3771.8 | uss_MB: 3001.3 | child_rss_MB: 4011.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:40:43,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:24,879] INFO : Step 12000/50000 rank0 | Train Loss: 9.4193 | Train Loss Unweighted: 3.2123 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8391 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1612 | Regression_eta: 0.0186 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9236 | Classification_binary: 2.8391 | Regression_cos_phi: 0.5523 | Regression_energy: 1.7665 | Regression_eta: 0.9149 | Regression_pt: 1.8833 | Regression_sin_phi: 0.5397 | LR: 8.78e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5838s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0030 | max_backward: 0.3080 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.3871 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3771.8 | uss_MB: 3001.4 | child_rss_MB: 4007.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:41:25,093] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:25,112] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 12:41:25,394] INFO : Running validation on rank0 +rank0 [2026-10-28 12:41:25,394] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:41:25,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:41:26,798] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 12:41:26,943] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:41:27,074] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:41:27,148] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:41:27,217] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:41:27,287] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:41:27,356] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:41:27,426] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:41:27,534] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:41:27,665] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:41:27,796] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:41:27,867] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:41:27,937] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:41:28,005] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:41:28,074] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:41:28,143] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:41:28,247] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:41:28,254] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:41:28,257] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:41:28,260] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:41:28,263] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:41:28,264] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:41:28,267] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:41:28,269] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:41:28,270] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:41:28,273] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:41:28,274] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:41:28,277] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:41:28,280] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:41:28,283] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:41:28,285] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:41:28,286] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:41:35,738] INFO : VALIDATION | Step=12000/50000 | Train Loss=9.4193 | Valid Loss=8.8975 | Train Loss Unweighted=3.2123 | Valid Loss Unweighted=3.1692 | Stale=0 | ETA=282.0m +rank0 [2026-10-28 12:41:35,748] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:35,827] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:41:35,830] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:35,983] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:37,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:38,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:39,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:39,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:39,999] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:41:40,003] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:40,094] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:41:40,109] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:40,266] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:42,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:42,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:43,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:43,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:44,057] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:44,060] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:41:44,137] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:41:44,140] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:41:44,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:41:46,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:41:46,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:41:47,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:41:48,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:41:48,154] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:41:48,162] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:48,425] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:41:48,827] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:50,723] INFO : Plotted jets +rank0 [2026-10-28 12:41:51,801] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:51,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:52,067] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:41:52,454] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:53,838] INFO : Plotted jets +rank0 [2026-10-28 12:41:54,945] INFO : Plotted jet ratio +rank0 [2026-10-28 12:41:54,978] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:55,185] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_12000/cld_edm_qq_pf +rank0 [2026-10-28 12:41:55,547] INFO : Plotted number of elements +rank0 [2026-10-28 12:41:56,920] INFO : Plotted jets +rank0 [2026-10-28 12:41:57,974] INFO : Plotted jet ratio +rank0 [2026-10-28 12:42:39,065] INFO : Step 12100/50000 rank0 | Train Loss: 9.0010 | Train Loss Unweighted: 3.2092 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.8786 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1384 | Regression_eta: 0.0138 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9499 | Classification_binary: 2.8786 | Regression_cos_phi: 0.8022 | Regression_energy: 1.5165 | Regression_eta: 0.6782 | Regression_pt: 1.6584 | Regression_sin_phi: 0.5172 | LR: 8.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5298s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.1859 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1203 | max_tensorboard_logging: 0.3013 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3842.8 | uss_MB: 3074.3 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:42:39,278] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:19,697] INFO : Step 12200/50000 rank0 | Train Loss: 9.3494 | Train Loss Unweighted: 3.2448 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.8844 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1572 | Regression_eta: 0.0212 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9104 | Classification_binary: 2.8844 | Regression_cos_phi: 0.4815 | Regression_energy: 1.7224 | Regression_eta: 1.0425 | Regression_pt: 1.7617 | Regression_sin_phi: 0.5465 | LR: 8.74e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5839s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3040 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 3843.8 | uss_MB: 3075.5 | child_rss_MB: 4009.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:43:19,913] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:01,283] INFO : Step 12300/50000 rank0 | Train Loss: 8.7551 | Train Loss Unweighted: 3.1029 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7495 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1535 | Regression_eta: 0.0103 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0209 | Classification_binary: 2.7495 | Regression_cos_phi: 0.4482 | Regression_energy: 1.6818 | Regression_eta: 0.5062 | Regression_pt: 1.8358 | Regression_sin_phi: 0.5126 | LR: 8.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6897s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1040 | max_loss: 0.0033 | max_backward: 0.3692 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.3812 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3843.8 | uss_MB: 3077.8 | child_rss_MB: 4012.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:44:01,498] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:42,544] INFO : Step 12400/50000 rank0 | Train Loss: 8.9929 | Train Loss Unweighted: 3.2015 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.8457 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1555 | Regression_eta: 0.0137 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0387 | Classification_binary: 2.8457 | Regression_cos_phi: 0.4955 | Regression_energy: 1.7034 | Regression_eta: 0.6756 | Regression_pt: 1.8029 | Regression_sin_phi: 0.4311 | LR: 8.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7646s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0283 | max_forward: 0.1040 | max_loss: 0.0033 | max_backward: 0.3930 | max_optimizer_step: 0.1181 | max_tensorboard_logging: 0.3959 | max_tensorboard_flush: 0.0058 | Memory: rss_MB: 3843.8 | uss_MB: 3077.7 | child_rss_MB: 4015.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:44:42,755] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:22,809] INFO : Step 12500/50000 rank0 | Train Loss: 8.5330 | Train Loss Unweighted: 3.1452 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.7928 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1573 | Regression_eta: 0.0091 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8685 | Classification_binary: 2.7928 | Regression_cos_phi: 0.4293 | Regression_energy: 1.7238 | Regression_eta: 0.4497 | Regression_pt: 1.8525 | Regression_sin_phi: 0.4164 | LR: 8.68e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5909s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2757 | max_optimizer_step: 0.1202 | max_tensorboard_logging: 0.3897 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3843.8 | uss_MB: 3076.0 | child_rss_MB: 4014.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:45:23,023] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:03,291] INFO : Step 12600/50000 rank0 | Train Loss: 8.6002 | Train Loss Unweighted: 3.1142 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7823 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1426 | Regression_eta: 0.0113 | Regression_pt: 0.1411 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0279 | Classification_binary: 2.7823 | Regression_cos_phi: 0.5066 | Regression_energy: 1.5628 | Regression_eta: 0.5557 | Regression_pt: 1.6927 | Regression_sin_phi: 0.4724 | LR: 8.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5545s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0612 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1586 | max_tensorboard_logging: 0.3210 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 3843.8 | uss_MB: 3076.0 | child_rss_MB: 4015.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:46:03,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:45,480] INFO : Step 12700/50000 rank0 | Train Loss: 8.5151 | Train Loss Unweighted: 3.1102 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7829 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1427 | Regression_eta: 0.0124 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0210 | Classification_binary: 2.7829 | Regression_cos_phi: 0.4765 | Regression_energy: 1.5634 | Regression_eta: 0.6105 | Regression_pt: 1.6397 | Regression_sin_phi: 0.4210 | LR: 8.64e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6739s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.3093 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.4066 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3843.1 | uss_MB: 3074.3 | child_rss_MB: 4015.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:46:45,693] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:26,561] INFO : Step 12800/50000 rank0 | Train Loss: 8.8267 | Train Loss Unweighted: 3.1154 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7680 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1532 | Regression_eta: 0.0112 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9373 | Classification_binary: 2.7680 | Regression_cos_phi: 0.6415 | Regression_energy: 1.6791 | Regression_eta: 0.5525 | Regression_pt: 1.7472 | Regression_sin_phi: 0.5012 | LR: 8.62e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4890s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0377 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.2026 | max_optimizer_step: 0.1756 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3845.1 | uss_MB: 3075.5 | child_rss_MB: 4017.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:47:26,791] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:08,048] INFO : Step 12900/50000 rank0 | Train Loss: 8.8230 | Train Loss Unweighted: 3.1213 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7778 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1483 | Regression_eta: 0.0138 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9264 | Classification_binary: 2.7778 | Regression_cos_phi: 0.5238 | Regression_energy: 1.6244 | Regression_eta: 0.6770 | Regression_pt: 1.7439 | Regression_sin_phi: 0.5496 | LR: 8.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5131s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0503 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.2033 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0143 | Memory: rss_MB: 3845.1 | uss_MB: 3075.6 | child_rss_MB: 4015.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:48:08,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:49,543] INFO : Step 13000/50000 rank0 | Train Loss: 8.9021 | Train Loss Unweighted: 3.1906 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.8431 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1511 | Regression_eta: 0.0117 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9532 | Classification_binary: 2.8431 | Regression_cos_phi: 0.4663 | Regression_energy: 1.6559 | Regression_eta: 0.5772 | Regression_pt: 1.7676 | Regression_sin_phi: 0.6388 | LR: 8.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0800 | max_backward: 0.2875 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.3044 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3845.1 | uss_MB: 3077.8 | child_rss_MB: 4018.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:48:49,756] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:49,775] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 12:48:49,951] INFO : Running validation on rank0 +rank0 [2026-10-28 12:48:49,951] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:48:49,951] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:48:51,265] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 12:48:51,411] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:48:51,542] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:48:51,617] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:48:51,686] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:48:51,754] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:48:51,823] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:48:51,892] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:48:52,002] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:48:52,133] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:48:52,264] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:48:52,334] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:48:52,403] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:48:52,471] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:48:52,540] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:48:52,610] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:48:52,732] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:48:52,739] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:48:52,742] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:48:52,745] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:48:52,746] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:48:52,749] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:48:52,751] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:48:52,752] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:48:52,755] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:48:52,758] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:48:52,759] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:48:52,762] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:48:52,765] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:48:52,766] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:48:52,768] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:48:52,769] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:49:00,227] INFO : VALIDATION | Step=13000/50000 | Train Loss=8.9021 | Valid Loss=8.8012 | Train Loss Unweighted=3.1906 | Valid Loss Unweighted=3.1547 | Stale=0 | ETA=274.5m +rank0 [2026-10-28 12:49:00,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:00,299] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:49:00,303] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:49:00,456] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:02,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:03,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:03,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:04,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:04,535] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:49:04,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:04,597] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:49:04,600] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:04,683] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:06,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:07,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:07,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:08,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:08,667] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:49:08,670] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:08,726] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:49:08,729] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:49:08,819] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:10,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:11,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:11,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:12,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:12,702] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:49:12,709] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:49:12,964] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:49:13,370] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:15,299] INFO : Plotted jets +rank0 [2026-10-28 12:49:16,377] INFO : Plotted jet ratio +rank0 [2026-10-28 12:49:16,411] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:16,638] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:17,024] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:18,410] INFO : Plotted jets +rank0 [2026-10-28 12:49:19,513] INFO : Plotted jet ratio +rank0 [2026-10-28 12:49:19,571] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 12:49:19,793] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_13000/cld_edm_qq_pf +rank0 [2026-10-28 12:49:20,154] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:21,529] INFO : Plotted jets +rank0 [2026-10-28 12:49:22,579] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:04,359] INFO : Step 13100/50000 rank0 | Train Loss: 9.1622 | Train Loss Unweighted: 3.1034 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.7317 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1631 | Regression_eta: 0.0142 | Regression_pt: 0.1569 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0181 | Classification_binary: 2.7317 | Regression_cos_phi: 0.5608 | Regression_energy: 1.7868 | Regression_eta: 0.6958 | Regression_pt: 1.8817 | Regression_sin_phi: 0.4875 | LR: 8.56e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5991s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2469 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3901 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4051.5 | uss_MB: 3283.1 | child_rss_MB: 4012.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:50:04,572] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:46,631] INFO : Step 13200/50000 rank0 | Train Loss: 10.1645 | Train Loss Unweighted: 3.2709 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8538 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1850 | Regression_eta: 0.0216 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9236 | Classification_binary: 2.8538 | Regression_cos_phi: 0.6024 | Regression_energy: 2.0268 | Regression_eta: 1.0640 | Regression_pt: 2.0621 | Regression_sin_phi: 0.6318 | LR: 8.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0655 | max_backward: 0.2028 | max_optimizer_step: 0.1194 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3918.0 | uss_MB: 3149.4 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:50:46,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:27,809] INFO : Step 13300/50000 rank0 | Train Loss: 8.6329 | Train Loss Unweighted: 3.0879 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7406 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1522 | Regression_eta: 0.0117 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9176 | Classification_binary: 2.7406 | Regression_cos_phi: 0.4419 | Regression_energy: 1.6671 | Regression_eta: 0.5772 | Regression_pt: 1.7925 | Regression_sin_phi: 0.4959 | LR: 8.52e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.7605s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.2053 | max_loss: 0.0032 | max_backward: 0.2868 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.3091 | max_tensorboard_flush: 0.0132 | Memory: rss_MB: 3918.0 | uss_MB: 3152.1 | child_rss_MB: 4012.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:51:28,025] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:08,453] INFO : Step 13400/50000 rank0 | Train Loss: 8.5197 | Train Loss Unweighted: 3.2077 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.8697 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1496 | Regression_eta: 0.0101 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9050 | Classification_binary: 2.8697 | Regression_cos_phi: 0.4663 | Regression_energy: 1.6388 | Regression_eta: 0.4968 | Regression_pt: 1.7495 | Regression_sin_phi: 0.3935 | LR: 8.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5417s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0158 | max_backward: 0.2027 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.3981 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3918.0 | uss_MB: 3150.1 | child_rss_MB: 4012.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:52:08,668] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:48,696] INFO : Step 13500/50000 rank0 | Train Loss: 9.5619 | Train Loss Unweighted: 3.2416 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8633 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1655 | Regression_eta: 0.0152 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9946 | Classification_binary: 2.8633 | Regression_cos_phi: 0.6292 | Regression_energy: 1.8136 | Regression_eta: 0.7477 | Regression_pt: 1.8869 | Regression_sin_phi: 0.6265 | LR: 8.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5339s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1851 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1180 | max_tensorboard_logging: 0.2984 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3918.0 | uss_MB: 3150.1 | child_rss_MB: 4010.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:52:48,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:29,364] INFO : Step 13600/50000 rank0 | Train Loss: 8.0709 | Train Loss Unweighted: 3.0367 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.7102 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1451 | Regression_eta: 0.0097 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8772 | Classification_binary: 2.7102 | Regression_cos_phi: 0.4050 | Regression_energy: 1.5899 | Regression_eta: 0.4780 | Regression_pt: 1.7056 | Regression_sin_phi: 0.3051 | LR: 8.45e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8253s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0605 | max_forward: 0.1038 | max_loss: 0.0030 | max_backward: 0.3944 | max_optimizer_step: 0.1199 | max_tensorboard_logging: 0.3984 | max_tensorboard_flush: 0.0145 | Memory: rss_MB: 3918.0 | uss_MB: 3152.1 | child_rss_MB: 4011.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:53:29,577] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:10,640] INFO : Step 13700/50000 rank0 | Train Loss: 8.2451 | Train Loss Unweighted: 3.0347 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.7217 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1352 | Regression_eta: 0.0104 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9106 | Classification_binary: 2.7217 | Regression_cos_phi: 0.5739 | Regression_energy: 1.4809 | Regression_eta: 0.5110 | Regression_pt: 1.5873 | Regression_sin_phi: 0.4597 | LR: 8.43e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7538s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1043 | max_loss: 0.0031 | max_backward: 0.3944 | max_optimizer_step: 0.1196 | max_tensorboard_logging: 0.3999 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 3918.0 | uss_MB: 3152.2 | child_rss_MB: 4012.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 12:54:10,854] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:51,013] INFO : Step 13800/50000 rank0 | Train Loss: 8.9969 | Train Loss Unweighted: 3.1875 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.8209 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1621 | Regression_eta: 0.0117 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0126 | Classification_binary: 2.8209 | Regression_cos_phi: 0.4949 | Regression_energy: 1.7764 | Regression_eta: 0.5769 | Regression_pt: 1.8832 | Regression_sin_phi: 0.4320 | LR: 8.41e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4986s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0449 | max_forward: 0.0772 | max_loss: 0.0030 | max_backward: 0.2020 | max_optimizer_step: 0.1763 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3918.0 | uss_MB: 3149.5 | child_rss_MB: 4008.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:54:51,240] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:32,142] INFO : Step 13900/50000 rank0 | Train Loss: 8.9735 | Train Loss Unweighted: 3.1339 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7748 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1596 | Regression_eta: 0.0139 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9382 | Classification_binary: 2.7748 | Regression_cos_phi: 0.5937 | Regression_energy: 1.7485 | Regression_eta: 0.6811 | Regression_pt: 1.8012 | Regression_sin_phi: 0.4360 | LR: 8.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6073s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0030 | max_backward: 0.3061 | max_optimizer_step: 0.1180 | max_tensorboard_logging: 0.3870 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3918.0 | uss_MB: 3149.4 | child_rss_MB: 4010.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:55:32,354] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:13,305] INFO : Step 14000/50000 rank0 | Train Loss: 8.5783 | Train Loss Unweighted: 2.9954 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6524 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1498 | Regression_eta: 0.0116 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9289 | Classification_binary: 2.6524 | Regression_cos_phi: 0.4507 | Regression_energy: 1.6415 | Regression_eta: 0.5726 | Regression_pt: 1.7495 | Regression_sin_phi: 0.5827 | LR: 8.37e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6537s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2087 | max_loss: 0.0032 | max_backward: 0.2863 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3919.0 | uss_MB: 3152.1 | child_rss_MB: 4011.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 12:56:13,535] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:13,555] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 12:56:13,738] INFO : Running validation on rank0 +rank0 [2026-10-28 12:56:13,738] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:56:13,738] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:56:15,102] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 12:56:15,248] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:56:15,383] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:56:15,458] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:56:15,529] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:56:15,599] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:56:15,669] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:56:15,739] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:56:15,848] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:56:15,979] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:56:16,112] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:56:16,183] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:56:16,254] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:56:16,324] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:56:16,394] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:56:16,464] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:56:16,565] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:56:16,571] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:56:16,573] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:56:16,576] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:56:16,577] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:56:16,581] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:56:16,583] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:56:16,586] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:56:16,587] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:56:16,590] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:56:16,591] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:56:16,593] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:56:16,596] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:56:16,600] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:56:16,602] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:56:16,605] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:56:24,351] INFO : VALIDATION | Step=14000/50000 | Train Loss=8.5783 | Valid Loss=8.7057 | Train Loss Unweighted=2.9954 | Valid Loss Unweighted=3.1052 | Stale=0 | ETA=267.0m +rank0 [2026-10-28 12:56:24,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:24,483] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:56:24,487] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 12:56:24,682] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:26,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:27,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:27,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:28,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:28,793] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:56:28,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:28,868] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:56:28,872] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:28,947] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:30,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:31,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:32,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:32,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:32,861] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:56:32,864] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:32,918] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 12:56:32,921] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 12:56:33,026] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:34,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:35,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:36,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:36,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:36,974] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:56:36,981] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:56:37,238] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-10-28 12:56:37,643] INFO : Plotted number of elements +rank0 [2026-10-28 12:56:39,578] INFO : Plotted jets +rank0 [2026-10-28 12:56:40,660] INFO : Plotted jet ratio +rank0 [2026-10-28 12:56:40,691] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:40,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:41,311] INFO : Plotted number of elements +rank0 [2026-10-28 12:56:42,711] INFO : Plotted jets +rank0 [2026-10-28 12:56:43,818] INFO : Plotted jet ratio +rank0 [2026-10-28 12:56:43,850] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 12:56:44,058] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_14000/cld_edm_qq_pf +rank0 [2026-10-28 12:56:44,421] INFO : Plotted number of elements +rank0 [2026-10-28 12:56:45,801] INFO : Plotted jets +rank0 [2026-10-28 12:56:46,864] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:27,990] INFO : Step 14100/50000 rank0 | Train Loss: 8.3070 | Train Loss Unweighted: 3.0408 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.7105 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1455 | Regression_eta: 0.0107 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9159 | Classification_binary: 2.7105 | Regression_cos_phi: 0.4309 | Regression_energy: 1.5938 | Regression_eta: 0.5281 | Regression_pt: 1.6964 | Regression_sin_phi: 0.4314 | LR: 8.35e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6098s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2521 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.3945 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4136.1 | uss_MB: 3369.9 | child_rss_MB: 4008.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:57:28,203] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:08,699] INFO : Step 14200/50000 rank0 | Train Loss: 9.0817 | Train Loss Unweighted: 3.0015 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6615 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1449 | Regression_eta: 0.0246 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8920 | Classification_binary: 2.6615 | Regression_cos_phi: 0.6007 | Regression_energy: 1.5877 | Regression_eta: 1.2069 | Regression_pt: 1.6134 | Regression_sin_phi: 0.5196 | LR: 8.32e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5684s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2955 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.3723 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3990.1 | uss_MB: 3223.5 | child_rss_MB: 4012.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:58:08,914] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:48,952] INFO : Step 14300/50000 rank0 | Train Loss: 8.0664 | Train Loss Unweighted: 3.0189 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6956 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1429 | Regression_eta: 0.0088 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9264 | Classification_binary: 2.6956 | Regression_cos_phi: 0.4059 | Regression_energy: 1.5662 | Regression_eta: 0.4337 | Regression_pt: 1.6808 | Regression_sin_phi: 0.3579 | LR: 8.30e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4163s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0934 | max_loss: 0.0036 | max_backward: 0.2034 | max_optimizer_step: 0.1171 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3990.1 | uss_MB: 3223.8 | child_rss_MB: 4011.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:58:49,166] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:29,715] INFO : Step 14400/50000 rank0 | Train Loss: 9.2970 | Train Loss Unweighted: 3.0879 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.7161 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1644 | Regression_eta: 0.0204 | Regression_pt: 0.1512 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9820 | Classification_binary: 2.7161 | Regression_cos_phi: 0.5113 | Regression_energy: 1.8010 | Regression_eta: 1.0024 | Regression_pt: 1.8127 | Regression_sin_phi: 0.4714 | LR: 8.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5067s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1844 | max_loss: 0.0031 | max_backward: 0.2025 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3990.1 | uss_MB: 3223.8 | child_rss_MB: 4013.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:59:29,931] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:10,238] INFO : Step 14500/50000 rank0 | Train Loss: 8.7149 | Train Loss Unweighted: 3.1416 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.7848 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1561 | Regression_eta: 0.0117 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9309 | Classification_binary: 2.7848 | Regression_cos_phi: 0.4020 | Regression_energy: 1.7102 | Regression_eta: 0.5745 | Regression_pt: 1.8736 | Regression_sin_phi: 0.4389 | LR: 8.26e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6171s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0032 | max_backward: 0.3110 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3990.1 | uss_MB: 3223.8 | child_rss_MB: 4008.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:00:10,450] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:50,640] INFO : Step 14600/50000 rank0 | Train Loss: 8.4265 | Train Loss Unweighted: 3.1238 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.7807 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1536 | Regression_eta: 0.0105 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8914 | Classification_binary: 2.7807 | Regression_cos_phi: 0.3978 | Regression_energy: 1.6833 | Regression_eta: 0.5151 | Regression_pt: 1.7758 | Regression_sin_phi: 0.3823 | LR: 8.24e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5110s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1741 | max_loss: 0.0032 | max_backward: 0.2025 | max_optimizer_step: 0.1182 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3990.1 | uss_MB: 3223.9 | child_rss_MB: 4009.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:00:50,858] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:31,979] INFO : Step 14700/50000 rank0 | Train Loss: 8.8976 | Train Loss Unweighted: 3.1662 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.8005 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1615 | Regression_eta: 0.0109 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8995 | Classification_binary: 2.8005 | Regression_cos_phi: 0.5254 | Regression_energy: 1.7701 | Regression_eta: 0.5341 | Regression_pt: 1.9078 | Regression_sin_phi: 0.4603 | LR: 8.21e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5925s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1047 | max_loss: 0.0531 | max_backward: 0.2862 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.2923 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3990.1 | uss_MB: 3226.5 | child_rss_MB: 4015.2 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 13:01:32,192] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:13,809] INFO : Step 14800/50000 rank0 | Train Loss: 8.6269 | Train Loss Unweighted: 3.1356 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.8058 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1422 | Regression_eta: 0.0128 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9223 | Classification_binary: 2.8058 | Regression_cos_phi: 0.5211 | Regression_energy: 1.5579 | Regression_eta: 0.6283 | Regression_pt: 1.6712 | Regression_sin_phi: 0.5203 | LR: 8.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1533 | max_loss: 0.0034 | max_backward: 0.2863 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3990.5 | uss_MB: 3225.8 | child_rss_MB: 4013.4 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 13:02:14,038] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:54,497] INFO : Step 14900/50000 rank0 | Train Loss: 9.2290 | Train Loss Unweighted: 3.2223 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8366 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1710 | Regression_eta: 0.0108 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9667 | Classification_binary: 2.8366 | Regression_cos_phi: 0.5257 | Regression_energy: 1.8741 | Regression_eta: 0.5286 | Regression_pt: 2.0133 | Regression_sin_phi: 0.4841 | LR: 8.17e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5177s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3990.5 | uss_MB: 3224.0 | child_rss_MB: 4011.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:02:54,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:35,766] INFO : Step 15000/50000 rank0 | Train Loss: 9.0036 | Train Loss Unweighted: 3.0300 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6785 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1491 | Regression_eta: 0.0178 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0021 | Classification_binary: 2.6785 | Regression_cos_phi: 0.5335 | Regression_energy: 1.6339 | Regression_eta: 0.8768 | Regression_pt: 1.7663 | Regression_sin_phi: 0.5126 | LR: 8.15e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6193s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3108 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3990.5 | uss_MB: 3224.0 | child_rss_MB: 4008.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:03:35,980] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:35,999] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 13:03:36,275] INFO : Running validation on rank0 +rank0 [2026-10-28 13:03:36,275] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:03:36,275] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:03:37,664] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 13:03:37,809] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:03:37,940] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:03:38,015] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:03:38,083] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:03:38,152] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:03:38,221] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:03:38,290] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:03:38,400] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:03:38,532] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:03:38,664] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:03:38,735] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:03:38,805] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:03:38,875] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:03:38,946] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:03:39,017] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:03:39,127] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:03:39,134] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:03:39,136] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:03:39,139] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:03:39,140] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:03:39,143] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:03:39,146] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:03:39,148] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:03:39,150] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:03:39,154] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:03:39,154] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:03:39,157] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:03:39,160] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:03:39,163] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:03:39,166] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:03:39,167] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:03:46,614] INFO : VALIDATION | Step=15000/50000 | Train Loss=9.0036 | Valid Loss=8.7900 | Train Loss Unweighted=3.0300 | Valid Loss Unweighted=3.1191 | Stale=1 | ETA=259.5m +rank0 [2026-10-28 13:03:46,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:46,739] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:03:46,742] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:03:46,946] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:48,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:49,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:50,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:50,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:50,973] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:03:50,976] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:51,054] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:03:51,057] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:03:51,124] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:53,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:53,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:54,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:54,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:55,006] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:03:55,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:55,081] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:03:55,084] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:03:55,177] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:56,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:57,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:58,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:59,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:59,125] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:03:59,133] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:03:59,394] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:03:59,808] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:01,245] INFO : Plotted jets +rank0 [2026-10-28 13:04:02,835] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:02,867] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:03,102] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:03,490] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:04,879] INFO : Plotted jets +rank0 [2026-10-28 13:04:05,985] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:06,017] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 13:04:06,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_15000/cld_edm_qq_pf +rank0 [2026-10-28 13:04:06,584] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:07,958] INFO : Plotted jets +rank0 [2026-10-28 13:04:09,019] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:51,101] INFO : Step 15100/50000 rank0 | Train Loss: 8.3137 | Train Loss Unweighted: 3.1305 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.7994 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1469 | Regression_eta: 0.0104 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9362 | Classification_binary: 2.7994 | Regression_cos_phi: 0.4070 | Regression_energy: 1.6091 | Regression_eta: 0.5113 | Regression_pt: 1.7082 | Regression_sin_phi: 0.3425 | LR: 8.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1099 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4196.1 | uss_MB: 3428.5 | child_rss_MB: 4008.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:04:51,326] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:32,321] INFO : Step 15200/50000 rank0 | Train Loss: 8.6116 | Train Loss Unweighted: 3.0664 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.7244 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1497 | Regression_eta: 0.0132 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9641 | Classification_binary: 2.7244 | Regression_cos_phi: 0.4789 | Regression_energy: 1.6406 | Regression_eta: 0.6475 | Regression_pt: 1.7366 | Regression_sin_phi: 0.4194 | LR: 8.10e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5302s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1788 | max_loss: 0.0033 | max_backward: 0.2034 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.2901 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4008.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:05:32,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:14,491] INFO : Step 15300/50000 rank0 | Train Loss: 8.1949 | Train Loss Unweighted: 3.0739 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7564 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1383 | Regression_eta: 0.0120 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.7564 | Regression_cos_phi: 0.4643 | Regression_energy: 1.5159 | Regression_eta: 0.5910 | Regression_pt: 1.6314 | Regression_sin_phi: 0.3758 | LR: 8.08e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5454s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0567 | max_backward: 0.2035 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.3613 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:06:14,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:55,926] INFO : Step 15400/50000 rank0 | Train Loss: 9.5427 | Train Loss Unweighted: 3.2858 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.8852 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1795 | Regression_eta: 0.0149 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8925 | Classification_binary: 2.8852 | Regression_cos_phi: 0.5740 | Regression_energy: 1.9672 | Regression_eta: 0.7305 | Regression_pt: 2.0631 | Regression_sin_phi: 0.4303 | LR: 8.05e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3071 | max_optimizer_step: 0.1218 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4010.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:06:56,141] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:37,472] INFO : Step 15500/50000 rank0 | Train Loss: 8.2147 | Train Loss Unweighted: 3.0525 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.7418 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1345 | Regression_eta: 0.0120 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9323 | Classification_binary: 2.7418 | Regression_cos_phi: 0.4910 | Regression_energy: 1.4734 | Regression_eta: 0.5877 | Regression_pt: 1.5644 | Regression_sin_phi: 0.4241 | LR: 8.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5143s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1823 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4065.3 | uss_MB: 3298.9 | child_rss_MB: 4009.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:07:37,684] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:19,214] INFO : Step 15600/50000 rank0 | Train Loss: 8.5969 | Train Loss Unweighted: 3.0652 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.7274 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1479 | Regression_eta: 0.0110 | Regression_pt: 0.1433 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8948 | Classification_binary: 2.7274 | Regression_cos_phi: 0.6436 | Regression_energy: 1.6207 | Regression_eta: 0.5431 | Regression_pt: 1.7181 | Regression_sin_phi: 0.4494 | LR: 8.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7215s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2113 | max_loss: 0.0032 | max_backward: 0.2862 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.2788 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4065.3 | uss_MB: 3301.1 | child_rss_MB: 4009.8 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 96.9 +rank0 [2026-10-28 13:08:19,438] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:00,426] INFO : Step 15700/50000 rank0 | Train Loss: 8.8387 | Train Loss Unweighted: 3.0443 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6841 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1609 | Regression_eta: 0.0154 | Regression_pt: 0.1508 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8584 | Classification_binary: 2.6841 | Regression_cos_phi: 0.4512 | Regression_energy: 1.7635 | Regression_eta: 0.7590 | Regression_pt: 1.8089 | Regression_sin_phi: 0.5136 | LR: 7.98e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5383s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.2275 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3817 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4066.3 | uss_MB: 3299.1 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:09:00,639] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:40,910] INFO : Step 15800/50000 rank0 | Train Loss: 8.5959 | Train Loss Unweighted: 3.1063 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.7684 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1510 | Regression_eta: 0.0124 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9291 | Classification_binary: 2.7684 | Regression_cos_phi: 0.5266 | Regression_energy: 1.6549 | Regression_eta: 0.6080 | Regression_pt: 1.6819 | Regression_sin_phi: 0.4271 | LR: 7.96e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5871s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2933 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.3873 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4066.4 | uss_MB: 3298.8 | child_rss_MB: 4009.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 13:09:41,121] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:22,713] INFO : Step 15900/50000 rank0 | Train Loss: 8.5419 | Train Loss Unweighted: 3.0809 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.7518 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1418 | Regression_eta: 0.0124 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9083 | Classification_binary: 2.7518 | Regression_cos_phi: 0.5806 | Regression_energy: 1.5541 | Regression_eta: 0.6085 | Regression_pt: 1.6755 | Regression_sin_phi: 0.4630 | LR: 7.94e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6046s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1666 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4066.4 | uss_MB: 3299.0 | child_rss_MB: 4010.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:10:22,944] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:04,181] INFO : Step 16000/50000 rank0 | Train Loss: 8.0822 | Train Loss Unweighted: 2.8681 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5479 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1396 | Regression_eta: 0.0121 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9008 | Classification_binary: 2.5479 | Regression_cos_phi: 0.5154 | Regression_energy: 1.5292 | Regression_eta: 0.5929 | Regression_pt: 1.6316 | Regression_sin_phi: 0.3645 | LR: 7.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5273s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0783 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.2821 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4066.4 | uss_MB: 3299.0 | child_rss_MB: 4011.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:11:04,410] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:04,429] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 13:11:04,600] INFO : Running validation on rank0 +rank0 [2026-10-28 13:11:04,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:11:04,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:11:05,925] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 13:11:06,070] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:11:06,201] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:11:06,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:11:06,345] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:11:06,414] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:11:06,483] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:11:06,552] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:11:06,661] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:11:06,791] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:11:06,923] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:11:06,995] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:11:07,064] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:11:07,135] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:11:07,204] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:11:07,273] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:11:07,387] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:11:07,394] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:11:07,397] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:11:07,400] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:11:07,401] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:11:07,404] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:11:07,407] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:11:07,408] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:11:07,411] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:11:07,413] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:11:07,414] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:11:07,417] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:11:07,418] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:11:07,421] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:11:07,424] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:11:07,426] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:11:15,075] INFO : VALIDATION | Step=16000/50000 | Train Loss=8.0822 | Valid Loss=8.6015 | Train Loss Unweighted=2.8681 | Valid Loss Unweighted=3.0532 | Stale=0 | ETA=252.2m +rank0 [2026-10-28 13:11:15,084] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:15,142] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:11:15,146] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:11:15,301] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:17,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:17,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:18,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:19,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:19,439] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:11:19,442] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:19,498] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:11:19,500] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:19,592] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:21,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:22,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:22,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:23,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:23,678] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:11:23,681] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:23,736] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:11:23,739] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:11:23,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:25,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:26,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:26,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:27,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:27,753] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:11:27,760] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:11:28,017] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:11:28,422] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:29,851] INFO : Plotted jets +rank0 [2026-10-28 13:11:31,426] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:31,458] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:31,684] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:32,073] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:33,490] INFO : Plotted jets +rank0 [2026-10-28 13:11:34,595] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:34,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 13:11:34,836] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_16000/cld_edm_qq_pf +rank0 [2026-10-28 13:11:35,197] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:36,575] INFO : Plotted jets +rank0 [2026-10-28 13:11:37,619] INFO : Plotted jet ratio +rank0 [2026-10-28 13:12:19,199] INFO : Step 16100/50000 rank0 | Train Loss: 9.0595 | Train Loss Unweighted: 3.1293 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7254 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1814 | Regression_eta: 0.0079 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9610 | Classification_binary: 2.7254 | Regression_cos_phi: 0.4355 | Regression_energy: 1.9879 | Regression_eta: 0.3905 | Regression_pt: 2.1780 | Regression_sin_phi: 0.3812 | LR: 7.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6175s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2693 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4273.4 | uss_MB: 3507.1 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:12:19,412] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:59,642] INFO : Step 16200/50000 rank0 | Train Loss: 8.2823 | Train Loss Unweighted: 3.0601 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7381 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1412 | Regression_eta: 0.0096 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9543 | Classification_binary: 2.7381 | Regression_cos_phi: 0.5232 | Regression_energy: 1.5469 | Regression_eta: 0.4712 | Regression_pt: 1.6407 | Regression_sin_phi: 0.4079 | LR: 7.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4910s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0423 | max_forward: 0.0781 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1632 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4139.2 | uss_MB: 3371.5 | child_rss_MB: 4006.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:12:59,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:40,649] INFO : Step 16300/50000 rank0 | Train Loss: 8.4116 | Train Loss Unweighted: 3.0740 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.7465 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1434 | Regression_eta: 0.0108 | Regression_pt: 0.1388 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9234 | Classification_binary: 2.7465 | Regression_cos_phi: 0.5246 | Regression_energy: 1.5707 | Regression_eta: 0.5297 | Regression_pt: 1.6645 | Regression_sin_phi: 0.4521 | LR: 7.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5812s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1221 | max_loss: 0.0033 | max_backward: 0.2033 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4139.2 | uss_MB: 3371.6 | child_rss_MB: 4005.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:13:40,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:22,738] INFO : Step 16400/50000 rank0 | Train Loss: 8.6753 | Train Loss Unweighted: 3.0365 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6754 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1609 | Regression_eta: 0.0131 | Regression_pt: 0.1549 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9421 | Classification_binary: 2.6754 | Regression_cos_phi: 0.4213 | Regression_energy: 1.7631 | Regression_eta: 0.6421 | Regression_pt: 1.8574 | Regression_sin_phi: 0.3737 | LR: 7.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6423s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0033 | max_backward: 0.3097 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3848 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4139.2 | uss_MB: 3371.7 | child_rss_MB: 4010.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:14:22,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:04,345] INFO : Step 16500/50000 rank0 | Train Loss: 8.9450 | Train Loss Unweighted: 2.9719 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5974 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1652 | Regression_eta: 0.0127 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9362 | Classification_binary: 2.5974 | Regression_cos_phi: 0.5817 | Regression_energy: 1.8104 | Regression_eta: 0.6226 | Regression_pt: 1.9277 | Regression_sin_phi: 0.4690 | LR: 7.79e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.6207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.3107 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4139.2 | uss_MB: 3371.8 | child_rss_MB: 4010.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:15:04,558] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:44,553] INFO : Step 16600/50000 rank0 | Train Loss: 8.7031 | Train Loss Unweighted: 3.0259 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6836 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1473 | Regression_eta: 0.0129 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9708 | Classification_binary: 2.6836 | Regression_cos_phi: 0.5514 | Regression_energy: 1.6138 | Regression_eta: 0.6331 | Regression_pt: 1.7435 | Regression_sin_phi: 0.5068 | LR: 7.77e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0034 | max_backward: 0.3112 | max_optimizer_step: 0.1228 | max_tensorboard_logging: 0.3847 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4139.2 | uss_MB: 3371.8 | child_rss_MB: 4009.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:15:44,764] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:27,081] INFO : Step 16700/50000 rank0 | Train Loss: 9.3254 | Train Loss Unweighted: 3.1841 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7857 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1774 | Regression_eta: 0.0119 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9624 | Classification_binary: 2.7857 | Regression_cos_phi: 0.4460 | Regression_energy: 1.9436 | Regression_eta: 0.5829 | Regression_pt: 2.0843 | Regression_sin_phi: 0.5204 | LR: 7.74e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5367s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0033 | max_backward: 0.2272 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4139.2 | uss_MB: 3371.9 | child_rss_MB: 4010.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 13:16:27,293] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:08,044] INFO : Step 16800/50000 rank0 | Train Loss: 9.0427 | Train Loss Unweighted: 3.0949 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.7374 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1551 | Regression_eta: 0.0167 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9031 | Classification_binary: 2.7374 | Regression_cos_phi: 0.5770 | Regression_energy: 1.6995 | Regression_eta: 0.8215 | Regression_pt: 1.7995 | Regression_sin_phi: 0.5046 | LR: 7.72e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0989 | max_loss: 0.0033 | max_backward: 0.2029 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4138.4 | uss_MB: 3371.6 | child_rss_MB: 4008.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 13:17:08,257] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:50,670] INFO : Step 16900/50000 rank0 | Train Loss: 8.6535 | Train Loss Unweighted: 3.0307 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7227 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1276 | Regression_eta: 0.0206 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9626 | Classification_binary: 2.7227 | Regression_cos_phi: 0.5511 | Regression_energy: 1.3986 | Regression_eta: 1.0148 | Regression_pt: 1.4714 | Regression_sin_phi: 0.5324 | LR: 7.69e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8296s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1068 | max_forward: 0.1048 | max_loss: 0.0033 | max_backward: 0.3952 | max_optimizer_step: 0.1207 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4138.4 | uss_MB: 3373.9 | child_rss_MB: 4010.0 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 13:17:50,882] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:32,305] INFO : Step 17000/50000 rank0 | Train Loss: 8.7193 | Train Loss Unweighted: 3.0443 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6958 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1533 | Regression_eta: 0.0168 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9446 | Classification_binary: 2.6958 | Regression_cos_phi: 0.4264 | Regression_energy: 1.6795 | Regression_eta: 0.8258 | Regression_pt: 1.7466 | Regression_sin_phi: 0.4006 | LR: 7.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6300s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0786 | max_loss: 0.0034 | max_backward: 0.3086 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.3902 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4138.4 | uss_MB: 3372.0 | child_rss_MB: 4011.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:18:32,519] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:32,538] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 13:18:32,817] INFO : Running validation on rank0 +rank0 [2026-10-28 13:18:32,817] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:18:32,817] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:18:34,218] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 13:18:34,363] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:18:34,497] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:18:34,571] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:18:34,640] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:18:34,710] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:18:34,779] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:18:34,850] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:18:34,960] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:18:35,093] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:18:35,224] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:18:35,296] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:18:35,365] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:18:35,433] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:18:35,503] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:18:35,574] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:18:35,684] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:18:35,691] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:18:35,694] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:18:35,697] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:18:35,700] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:18:35,701] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:18:35,704] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:18:35,707] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:18:35,708] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:18:35,710] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:18:35,711] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:18:35,714] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:18:35,717] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:18:35,718] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:18:35,720] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:18:35,723] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:18:43,213] INFO : VALIDATION | Step=17000/50000 | Train Loss=8.7193 | Valid Loss=8.4506 | Train Loss Unweighted=3.0443 | Valid Loss Unweighted=3.0090 | Stale=0 | ETA=244.9m +rank0 [2026-10-28 13:18:43,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:43,322] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:18:43,326] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:18:43,481] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:45,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:46,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:46,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:47,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:47,535] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:18:47,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:47,606] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:18:47,609] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:47,689] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:49,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:50,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:50,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:51,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:51,681] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:18:51,684] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:51,753] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:18:51,756] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:18:51,837] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:53,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:54,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:54,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:55,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:55,711] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:18:55,719] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:18:55,977] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:18:56,382] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:58,320] INFO : Plotted jets +rank0 [2026-10-28 13:18:59,405] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:59,439] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:59,668] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:19:00,059] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:01,450] INFO : Plotted jets +rank0 [2026-10-28 13:19:02,556] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:02,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:02,795] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_17000/cld_edm_qq_pf +rank0 [2026-10-28 13:19:03,160] INFO : Plotted number of elements +rank0 [2026-10-28 13:19:04,548] INFO : Plotted jets +rank0 [2026-10-28 13:19:05,603] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:47,105] INFO : Step 17100/50000 rank0 | Train Loss: 8.3476 | Train Loss Unweighted: 2.9992 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6652 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1475 | Regression_eta: 0.0114 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9343 | Classification_binary: 2.6652 | Regression_cos_phi: 0.4370 | Regression_energy: 1.6160 | Regression_eta: 0.5612 | Regression_pt: 1.7018 | Regression_sin_phi: 0.4320 | LR: 7.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6067s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1441 | max_loss: 0.0034 | max_backward: 0.2025 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.2922 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4348.4 | uss_MB: 3583.2 | child_rss_MB: 4008.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:19:47,332] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:28,999] INFO : Step 17200/50000 rank0 | Train Loss: 8.0936 | Train Loss Unweighted: 2.9793 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6702 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1346 | Regression_eta: 0.0114 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9661 | Classification_binary: 2.6702 | Regression_cos_phi: 0.4725 | Regression_energy: 1.4743 | Regression_eta: 0.5599 | Regression_pt: 1.5498 | Regression_sin_phi: 0.4008 | LR: 7.62e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3105 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.3706 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4210.6 | uss_MB: 3444.7 | child_rss_MB: 4007.4 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:20:29,212] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:09,537] INFO : Step 17300/50000 rank0 | Train Loss: 8.4364 | Train Loss Unweighted: 3.0296 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6768 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1575 | Regression_eta: 0.0115 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.6768 | Regression_cos_phi: 0.4129 | Regression_energy: 1.7259 | Regression_eta: 0.5662 | Regression_pt: 1.8441 | Regression_sin_phi: 0.3504 | LR: 7.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5243s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0510 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.2940 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4210.6 | uss_MB: 3444.9 | child_rss_MB: 4005.7 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:21:09,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:50,337] INFO : Step 17400/50000 rank0 | Train Loss: 8.6185 | Train Loss Unweighted: 3.0284 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6985 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1433 | Regression_eta: 0.0134 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9279 | Classification_binary: 2.6985 | Regression_cos_phi: 0.6097 | Regression_energy: 1.5698 | Regression_eta: 0.6567 | Regression_pt: 1.6343 | Regression_sin_phi: 0.5216 | LR: 7.57e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7458s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2051 | max_loss: 0.0032 | max_backward: 0.2864 | max_optimizer_step: 0.2175 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4210.6 | uss_MB: 3447.2 | child_rss_MB: 4004.1 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:21:50,569] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:31,806] INFO : Step 17500/50000 rank0 | Train Loss: 8.7239 | Train Loss Unweighted: 3.0728 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.7257 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1509 | Regression_eta: 0.0117 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9482 | Classification_binary: 2.7257 | Regression_cos_phi: 0.5798 | Regression_energy: 1.6530 | Regression_eta: 0.5761 | Regression_pt: 1.7819 | Regression_sin_phi: 0.4591 | LR: 7.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6158s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1504 | max_loss: 0.0033 | max_backward: 0.2867 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.2893 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4211.6 | uss_MB: 3447.2 | child_rss_MB: 4000.9 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 13:22:32,028] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:13,330] INFO : Step 17600/50000 rank0 | Train Loss: 7.9389 | Train Loss Unweighted: 2.9411 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6231 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1399 | Regression_eta: 0.0089 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8998 | Classification_binary: 2.6231 | Regression_cos_phi: 0.4321 | Regression_energy: 1.5325 | Regression_eta: 0.4359 | Regression_pt: 1.6557 | Regression_sin_phi: 0.3598 | LR: 7.52e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5149s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0440 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.2025 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.2859 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4211.6 | uss_MB: 3445.2 | child_rss_MB: 4001.6 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:23:13,560] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:55,206] INFO : Step 17700/50000 rank0 | Train Loss: 8.4270 | Train Loss Unweighted: 3.0586 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.7308 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1445 | Regression_eta: 0.0131 | Regression_pt: 0.1370 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8838 | Classification_binary: 2.7308 | Regression_cos_phi: 0.5019 | Regression_energy: 1.5830 | Regression_eta: 0.6439 | Regression_pt: 1.6435 | Regression_sin_phi: 0.4401 | LR: 7.49e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6206s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.3074 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4211.6 | uss_MB: 3445.2 | child_rss_MB: 4003.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:23:55,421] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:36,700] INFO : Step 17800/50000 rank0 | Train Loss: 8.0407 | Train Loss Unweighted: 2.9458 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.6244 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1416 | Regression_eta: 0.0101 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9029 | Classification_binary: 2.6244 | Regression_cos_phi: 0.4337 | Regression_energy: 1.5519 | Regression_eta: 0.4981 | Regression_pt: 1.6566 | Regression_sin_phi: 0.3731 | LR: 7.47e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5503s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.2378 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4211.5 | uss_MB: 3444.2 | child_rss_MB: 4003.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:24:36,911] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:17,542] INFO : Step 17900/50000 rank0 | Train Loss: 7.9466 | Train Loss Unweighted: 2.8611 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5574 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1325 | Regression_eta: 0.0130 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8278 | Classification_binary: 2.5574 | Regression_cos_phi: 0.5040 | Regression_energy: 1.4522 | Regression_eta: 0.6379 | Regression_pt: 1.5105 | Regression_sin_phi: 0.4568 | LR: 7.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6386s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.3111 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3905 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4212.5 | uss_MB: 3445.0 | child_rss_MB: 4000.8 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:25:17,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:57,870] INFO : Step 18000/50000 rank0 | Train Loss: 9.0771 | Train Loss Unweighted: 3.0841 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.7124 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1615 | Regression_eta: 0.0155 | Regression_pt: 0.1598 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9163 | Classification_binary: 2.7124 | Regression_cos_phi: 0.4795 | Regression_energy: 1.7697 | Regression_eta: 0.7630 | Regression_pt: 1.9166 | Regression_sin_phi: 0.5196 | LR: 7.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4841s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1510 | max_loss: 0.0032 | max_backward: 0.2020 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4212.5 | uss_MB: 3445.1 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:25:58,086] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:58,107] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 13:25:58,288] INFO : Running validation on rank0 +rank0 [2026-10-28 13:25:58,288] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:25:58,288] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:25:59,676] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 13:25:59,823] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:25:59,954] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:26:00,029] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:26:00,099] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:26:00,168] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:26:00,238] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:26:00,307] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:26:00,418] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:26:00,549] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:26:00,679] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:26:00,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:26:00,820] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:26:00,887] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:26:00,957] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:26:01,027] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:26:01,137] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:26:01,144] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:26:01,146] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:26:01,149] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:26:01,150] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:26:01,153] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:26:01,154] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:26:01,157] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:26:01,159] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:26:01,160] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:26:01,163] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:26:01,165] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:26:01,166] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:26:01,169] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:26:01,170] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:26:01,173] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:26:08,841] INFO : VALIDATION | Step=18000/50000 | Train Loss=9.0771 | Valid Loss=8.4217 | Train Loss Unweighted=3.0841 | Valid Loss Unweighted=2.9995 | Stale=0 | ETA=237.5m +rank0 [2026-10-28 13:26:08,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:08,926] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:26:08,930] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:09,093] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:11,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:11,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:12,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:13,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:13,311] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:26:13,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:13,394] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:26:13,397] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:13,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:15,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:15,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:16,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:17,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:17,442] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:26:17,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:26:17,526] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:26:17,530] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:26:17,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:26:19,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:26:20,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:26:20,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:26:21,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:26:21,553] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:26:21,561] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:21,817] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:26:22,220] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:24,130] INFO : Plotted jets +rank0 [2026-10-28 13:26:25,213] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:25,246] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:25,473] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:26:25,862] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:27,252] INFO : Plotted jets +rank0 [2026-10-28 13:26:28,358] INFO : Plotted jet ratio +rank0 [2026-10-28 13:26:28,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:28,603] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_18000/cld_edm_qq_pf +rank0 [2026-10-28 13:26:28,966] INFO : Plotted number of elements +rank0 [2026-10-28 13:26:30,343] INFO : Plotted jets +rank0 [2026-10-28 13:26:31,396] INFO : Plotted jet ratio +rank0 [2026-10-28 13:27:13,795] INFO : Step 18100/50000 rank0 | Train Loss: 8.0182 | Train Loss Unweighted: 2.9006 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5822 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1426 | Regression_eta: 0.0096 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9449 | Classification_binary: 2.5822 | Regression_cos_phi: 0.4793 | Regression_energy: 1.5621 | Regression_eta: 0.4700 | Regression_pt: 1.5944 | Regression_sin_phi: 0.3852 | LR: 7.39e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.6557s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.3084 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4427.3 | uss_MB: 3661.9 | child_rss_MB: 4007.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:27:14,005] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:27:55,277] INFO : Step 18200/50000 rank0 | Train Loss: 8.5521 | Train Loss Unweighted: 3.0129 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6804 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1445 | Regression_eta: 0.0128 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8832 | Classification_binary: 2.6804 | Regression_cos_phi: 0.5165 | Regression_energy: 1.5829 | Regression_eta: 0.6271 | Regression_pt: 1.6733 | Regression_sin_phi: 0.5887 | LR: 7.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.3097 | max_optimizer_step: 0.1246 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4283.5 | uss_MB: 3517.3 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:27:55,491] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:28:36,731] INFO : Step 18300/50000 rank0 | Train Loss: 8.4956 | Train Loss Unweighted: 2.9940 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6410 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1570 | Regression_eta: 0.0095 | Regression_pt: 0.1534 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8549 | Classification_binary: 2.6410 | Regression_cos_phi: 0.4901 | Regression_energy: 1.7205 | Regression_eta: 0.4672 | Regression_pt: 1.8392 | Regression_sin_phi: 0.4827 | LR: 7.34e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1060 | max_forward: 0.1039 | max_loss: 0.0032 | max_backward: 0.3929 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3904 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4284.5 | uss_MB: 3520.3 | child_rss_MB: 4007.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:28:36,946] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:17,798] INFO : Step 18400/50000 rank0 | Train Loss: 8.6009 | Train Loss Unweighted: 3.0080 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6734 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1478 | Regression_eta: 0.0154 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9629 | Classification_binary: 2.6734 | Regression_cos_phi: 0.4904 | Regression_energy: 1.6192 | Regression_eta: 0.7574 | Regression_pt: 1.6353 | Regression_sin_phi: 0.4624 | LR: 7.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5408s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1826 | max_loss: 0.0033 | max_backward: 0.2024 | max_optimizer_step: 0.1235 | max_tensorboard_logging: 0.2798 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4284.5 | uss_MB: 3518.4 | child_rss_MB: 4007.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:29:18,014] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:59,103] INFO : Step 18500/50000 rank0 | Train Loss: 8.0647 | Train Loss Unweighted: 2.9308 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6198 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1335 | Regression_eta: 0.0090 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9516 | Classification_binary: 2.6198 | Regression_cos_phi: 0.5532 | Regression_energy: 1.4624 | Regression_eta: 0.4406 | Regression_pt: 1.5993 | Regression_sin_phi: 0.4379 | LR: 7.29e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6184s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.2806 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3867 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4284.5 | uss_MB: 3518.5 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:29:59,314] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:30:40,886] INFO : Step 18600/50000 rank0 | Train Loss: 7.8086 | Train Loss Unweighted: 2.9032 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6069 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1276 | Regression_eta: 0.0086 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8893 | Classification_binary: 2.6069 | Regression_cos_phi: 0.4725 | Regression_energy: 1.3987 | Regression_eta: 0.4238 | Regression_pt: 1.5134 | Regression_sin_phi: 0.5040 | LR: 7.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5630s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1244 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4284.5 | uss_MB: 3518.6 | child_rss_MB: 4004.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 13:30:41,111] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:31:22,192] INFO : Step 18700/50000 rank0 | Train Loss: 8.4602 | Train Loss Unweighted: 2.9952 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6599 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1460 | Regression_eta: 0.0103 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8939 | Classification_binary: 2.6599 | Regression_cos_phi: 0.5413 | Regression_energy: 1.5992 | Regression_eta: 0.5050 | Regression_pt: 1.7214 | Regression_sin_phi: 0.5395 | LR: 7.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5398s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1854 | max_loss: 0.0033 | max_backward: 0.2026 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4284.5 | uss_MB: 3518.6 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:31:22,407] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:03,686] INFO : Step 18800/50000 rank0 | Train Loss: 8.4335 | Train Loss Unweighted: 2.8989 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5783 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1374 | Regression_eta: 0.0206 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8401 | Classification_binary: 2.5783 | Regression_cos_phi: 0.4429 | Regression_energy: 1.5054 | Regression_eta: 1.0118 | Regression_pt: 1.5642 | Regression_sin_phi: 0.4907 | LR: 7.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5503s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1848 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2981 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4282.6 | uss_MB: 3517.4 | child_rss_MB: 4007.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:32:03,910] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:44,551] INFO : Step 18900/50000 rank0 | Train Loss: 8.5882 | Train Loss Unweighted: 3.0248 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6732 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1575 | Regression_eta: 0.0113 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8828 | Classification_binary: 2.6732 | Regression_cos_phi: 0.5376 | Regression_energy: 1.7258 | Regression_eta: 0.5568 | Regression_pt: 1.7931 | Regression_sin_phi: 0.4189 | LR: 7.18e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6011s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2639 | max_optimizer_step: 0.1240 | max_tensorboard_logging: 0.3932 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4282.6 | uss_MB: 3518.2 | child_rss_MB: 3999.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:32:44,763] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:33:25,385] INFO : Step 19000/50000 rank0 | Train Loss: 8.2713 | Train Loss Unweighted: 3.0333 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.7099 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1421 | Regression_eta: 0.0107 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9244 | Classification_binary: 2.7099 | Regression_cos_phi: 0.5212 | Regression_energy: 1.5568 | Regression_eta: 0.5281 | Regression_pt: 1.6444 | Regression_sin_phi: 0.3865 | LR: 7.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6455s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0032 | max_backward: 0.3107 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.3946 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4282.6 | uss_MB: 3518.2 | child_rss_MB: 3999.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:33:25,596] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:33:25,708] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 13:33:25,886] INFO : Running validation on rank0 +rank0 [2026-10-28 13:33:25,886] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:33:25,886] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:33:27,214] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 13:33:27,360] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:33:27,491] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:33:27,565] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:33:27,634] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:33:27,703] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:33:27,773] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:33:27,842] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:33:27,954] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:33:28,085] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:33:28,216] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:33:28,287] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:33:28,356] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:33:28,424] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:33:28,494] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:33:28,562] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:33:28,672] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:33:28,679] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:33:28,682] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:33:28,684] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:33:28,686] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:33:28,689] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:33:28,692] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:33:28,693] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:33:28,696] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:33:28,697] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:33:28,700] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:33:28,702] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:33:28,703] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:33:28,706] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:33:28,708] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:33:28,709] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:33:36,311] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.2713 | Valid Loss=8.3138 | Train Loss Unweighted=3.0333 | Valid Loss Unweighted=2.9580 | Stale=0 | ETA=230.1m +rank0 [2026-10-28 13:33:36,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:33:36,384] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:33:36,387] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:33:36,546] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:33:38,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:33:39,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:33:39,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:33:40,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:33:40,555] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:33:40,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:33:40,699] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:33:40,702] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:33:40,862] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:33:42,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:33:43,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:33:44,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:33:44,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:33:44,784] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:33:44,787] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:33:44,879] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:33:44,892] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:33:45,058] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:33:46,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:33:47,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:33:48,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:33:48,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:33:48,930] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:33:48,938] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:33:49,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:33:49,598] INFO : Plotted number of elements +rank0 [2026-10-28 13:33:51,030] INFO : Plotted jets +rank0 [2026-10-28 13:33:52,635] INFO : Plotted jet ratio +rank0 [2026-10-28 13:33:52,669] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:33:52,901] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:33:53,289] INFO : Plotted number of elements +rank0 [2026-10-28 13:33:54,677] INFO : Plotted jets +rank0 [2026-10-28 13:33:55,789] INFO : Plotted jet ratio +rank0 [2026-10-28 13:33:55,825] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 13:33:56,036] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_19000/cld_edm_qq_pf +rank0 [2026-10-28 13:33:56,405] INFO : Plotted number of elements +rank0 [2026-10-28 13:33:57,779] INFO : Plotted jets +rank0 [2026-10-28 13:33:58,847] INFO : Plotted jet ratio +rank0 [2026-10-28 13:34:40,805] INFO : Step 19100/50000 rank0 | Train Loss: 8.3717 | Train Loss Unweighted: 3.0004 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6775 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1424 | Regression_eta: 0.0111 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9342 | Classification_binary: 2.6775 | Regression_cos_phi: 0.6129 | Regression_energy: 1.5606 | Regression_eta: 0.5474 | Regression_pt: 1.6010 | Regression_sin_phi: 0.4381 | LR: 7.13e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5816s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1817 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1261 | max_tensorboard_logging: 0.2966 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4483.8 | uss_MB: 3718.7 | child_rss_MB: 3998.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:34:41,020] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:35:21,801] INFO : Step 19200/50000 rank0 | Train Loss: 8.0526 | Train Loss Unweighted: 3.0095 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6838 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1448 | Regression_eta: 0.0091 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8346 | Classification_binary: 2.6838 | Regression_cos_phi: 0.4501 | Regression_energy: 1.5869 | Regression_eta: 0.4483 | Regression_pt: 1.7012 | Regression_sin_phi: 0.3478 | LR: 7.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7549s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2102 | max_loss: 0.0033 | max_backward: 0.2863 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.3150 | max_tensorboard_flush: 0.0135 | Memory: rss_MB: 4359.3 | uss_MB: 3595.6 | child_rss_MB: 3994.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:35:22,013] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:02,624] INFO : Step 19300/50000 rank0 | Train Loss: 8.2186 | Train Loss Unweighted: 2.9027 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5797 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1411 | Regression_eta: 0.0117 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8450 | Classification_binary: 2.5797 | Regression_cos_phi: 0.5185 | Regression_energy: 1.5460 | Regression_eta: 0.5770 | Regression_pt: 1.6362 | Regression_sin_phi: 0.5164 | LR: 7.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6707s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1810 | max_loss: 0.0033 | max_backward: 0.2868 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.2974 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4359.3 | uss_MB: 3596.7 | child_rss_MB: 3997.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:36:02,842] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:44,169] INFO : Step 19400/50000 rank0 | Train Loss: 8.6481 | Train Loss Unweighted: 3.0517 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7070 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1510 | Regression_eta: 0.0107 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0218 | Classification_binary: 2.7070 | Regression_cos_phi: 0.5424 | Regression_energy: 1.6551 | Regression_eta: 0.5260 | Regression_pt: 1.7543 | Regression_sin_phi: 0.4414 | LR: 7.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6144s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1656 | max_loss: 0.0032 | max_backward: 0.2024 | max_optimizer_step: 0.2246 | max_tensorboard_logging: 0.2779 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4359.3 | uss_MB: 3595.0 | child_rss_MB: 3999.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:36:44,393] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:37:24,989] INFO : Step 19500/50000 rank0 | Train Loss: 7.8863 | Train Loss Unweighted: 2.8800 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5720 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1342 | Regression_eta: 0.0097 | Regression_pt: 0.1316 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8782 | Classification_binary: 2.5720 | Regression_cos_phi: 0.5032 | Regression_energy: 1.4703 | Regression_eta: 0.4785 | Regression_pt: 1.5785 | Regression_sin_phi: 0.4057 | LR: 7.02e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.5112s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0461 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.2029 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4359.3 | uss_MB: 3595.1 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:37:25,215] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:38:06,429] INFO : Step 19600/50000 rank0 | Train Loss: 8.2227 | Train Loss Unweighted: 2.8989 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5863 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1338 | Regression_eta: 0.0155 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9282 | Classification_binary: 2.5863 | Regression_cos_phi: 0.4953 | Regression_energy: 1.4661 | Regression_eta: 0.7635 | Regression_pt: 1.5515 | Regression_sin_phi: 0.4319 | LR: 7.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5576s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0903 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.2034 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4359.3 | uss_MB: 3595.1 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:38:06,661] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:38:47,526] INFO : Step 19700/50000 rank0 | Train Loss: 8.5903 | Train Loss Unweighted: 2.9633 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6228 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1497 | Regression_eta: 0.0159 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8554 | Classification_binary: 2.6228 | Regression_cos_phi: 0.4766 | Regression_energy: 1.6406 | Regression_eta: 0.7834 | Regression_pt: 1.6966 | Regression_sin_phi: 0.5151 | LR: 6.97e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6493s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0033 | max_backward: 0.3060 | max_optimizer_step: 0.1276 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4359.3 | uss_MB: 3595.2 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:38:47,742] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:39:29,326] INFO : Step 19800/50000 rank0 | Train Loss: 9.3358 | Train Loss Unweighted: 2.9734 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6099 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1494 | Regression_eta: 0.0261 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9358 | Classification_binary: 2.6099 | Regression_cos_phi: 0.5500 | Regression_energy: 1.6365 | Regression_eta: 1.2815 | Regression_pt: 1.8262 | Regression_sin_phi: 0.4960 | LR: 6.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1707 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4357.4 | uss_MB: 3593.4 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:39:29,552] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:10,110] INFO : Step 19900/50000 rank0 | Train Loss: 8.1791 | Train Loss Unweighted: 2.8765 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5607 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1368 | Regression_eta: 0.0148 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8553 | Classification_binary: 2.5607 | Regression_cos_phi: 0.4987 | Regression_energy: 1.4988 | Regression_eta: 0.7298 | Regression_pt: 1.5748 | Regression_sin_phi: 0.4610 | LR: 6.92e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0682 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.1612 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4357.4 | uss_MB: 3594.8 | child_rss_MB: 3995.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:40:10,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:50,601] INFO : Step 20000/50000 rank0 | Train Loss: 9.3311 | Train Loss Unweighted: 3.0955 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7224 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1645 | Regression_eta: 0.0203 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8982 | Classification_binary: 2.7224 | Regression_cos_phi: 0.5752 | Regression_energy: 1.8020 | Regression_eta: 0.9989 | Regression_pt: 1.8335 | Regression_sin_phi: 0.5008 | LR: 6.89e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5339s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0448 | max_backward: 0.2031 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.3444 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4357.4 | uss_MB: 3594.9 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:40:50,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:50,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 13:40:51,035] INFO : Running validation on rank0 +rank0 [2026-10-28 13:40:51,035] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:40:51,035] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:40:52,333] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 13:40:52,479] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:40:52,611] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:40:52,685] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:40:52,755] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:40:52,823] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:40:52,893] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:40:52,962] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:40:53,076] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:40:53,208] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:40:53,339] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:40:53,410] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:40:53,480] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:40:53,548] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:40:53,617] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:40:53,686] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:40:53,795] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:40:53,802] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:40:53,805] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:40:53,808] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:40:53,810] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:40:53,811] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:40:53,814] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:40:53,817] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:40:53,819] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:40:53,821] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:40:53,823] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:40:53,824] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:40:53,827] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:40:53,830] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:40:53,832] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:40:53,834] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:41:01,275] INFO : VALIDATION | Step=20000/50000 | Train Loss=9.3311 | Valid Loss=8.2836 | Train Loss Unweighted=3.0955 | Valid Loss Unweighted=2.9411 | Stale=0 | ETA=222.7m +rank0 [2026-10-28 13:41:01,285] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:01,344] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:41:01,348] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:41:01,509] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:41:03,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:41:04,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:41:04,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:41:05,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:41:05,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:41:05,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:05,675] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:41:05,678] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:41:05,773] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:41:07,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:41:08,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:41:09,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:41:09,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:41:09,731] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:41:09,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:09,791] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:41:09,794] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:41:09,889] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:41:11,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:41:12,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:41:12,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:41:13,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:41:13,780] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:41:13,788] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:41:14,046] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:41:14,453] INFO : Plotted number of elements +rank0 [2026-10-28 13:41:15,898] INFO : Plotted jets +rank0 [2026-10-28 13:41:17,504] INFO : Plotted jet ratio +rank0 [2026-10-28 13:41:17,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:41:17,764] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:41:18,152] INFO : Plotted number of elements +rank0 [2026-10-28 13:41:19,538] INFO : Plotted jets +rank0 [2026-10-28 13:41:20,647] INFO : Plotted jet ratio +rank0 [2026-10-28 13:41:20,680] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 13:41:20,889] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_20000/cld_edm_qq_pf +rank0 [2026-10-28 13:41:21,251] INFO : Plotted number of elements +rank0 [2026-10-28 13:41:22,627] INFO : Plotted jets +rank0 [2026-10-28 13:41:23,689] INFO : Plotted jet ratio +rank0 [2026-10-28 13:42:04,446] INFO : Step 20100/50000 rank0 | Train Loss: 9.0107 | Train Loss Unweighted: 3.0371 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6994 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1460 | Regression_eta: 0.0205 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8264 | Classification_binary: 2.6994 | Regression_cos_phi: 0.8001 | Regression_energy: 1.5995 | Regression_eta: 1.0060 | Regression_pt: 1.6153 | Regression_sin_phi: 0.4640 | LR: 6.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6701s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3080 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3886 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4549.9 | uss_MB: 3789.3 | child_rss_MB: 3997.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:42:04,658] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:42:45,945] INFO : Step 20200/50000 rank0 | Train Loss: 8.2460 | Train Loss Unweighted: 2.9560 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.6185 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1498 | Regression_eta: 0.0097 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8676 | Classification_binary: 2.6185 | Regression_cos_phi: 0.4873 | Regression_energy: 1.6416 | Regression_eta: 0.4763 | Regression_pt: 1.7500 | Regression_sin_phi: 0.4046 | LR: 6.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6838s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3080 | max_optimizer_step: 0.1270 | max_tensorboard_logging: 0.4077 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4425.2 | uss_MB: 3664.2 | child_rss_MB: 3995.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:42:46,159] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:43:26,769] INFO : Step 20300/50000 rank0 | Train Loss: 8.5452 | Train Loss Unweighted: 3.0755 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.7369 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1480 | Regression_eta: 0.0107 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9492 | Classification_binary: 2.7369 | Regression_cos_phi: 0.5304 | Regression_energy: 1.6222 | Regression_eta: 0.5280 | Regression_pt: 1.7369 | Regression_sin_phi: 0.4416 | LR: 6.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5638s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.2195 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4425.2 | uss_MB: 3665.4 | child_rss_MB: 3993.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:43:26,981] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:07,157] INFO : Step 20400/50000 rank0 | Train Loss: 8.5537 | Train Loss Unweighted: 3.0628 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7267 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1469 | Regression_eta: 0.0111 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8571 | Classification_binary: 2.7267 | Regression_cos_phi: 0.5053 | Regression_energy: 1.6094 | Regression_eta: 0.5459 | Regression_pt: 1.7164 | Regression_sin_phi: 0.5929 | LR: 6.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1512 | max_loss: 0.0034 | max_backward: 0.2870 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.2882 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4425.2 | uss_MB: 3668.1 | child_rss_MB: 3998.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:44:07,379] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:47,644] INFO : Step 20500/50000 rank0 | Train Loss: 8.2588 | Train Loss Unweighted: 2.9325 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6106 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1403 | Regression_eta: 0.0139 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9249 | Classification_binary: 2.6106 | Regression_cos_phi: 0.4682 | Regression_energy: 1.5373 | Regression_eta: 0.6843 | Regression_pt: 1.6123 | Regression_sin_phi: 0.4212 | LR: 6.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.3090 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.3749 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4425.2 | uss_MB: 3666.0 | child_rss_MB: 3996.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:44:47,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:45:29,320] INFO : Step 20600/50000 rank0 | Train Loss: 7.5367 | Train Loss Unweighted: 2.8990 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6025 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1308 | Regression_eta: 0.0068 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8990 | Classification_binary: 2.6025 | Regression_cos_phi: 0.3784 | Regression_energy: 1.4331 | Regression_eta: 0.3341 | Regression_pt: 1.5433 | Regression_sin_phi: 0.3464 | LR: 6.73e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5529s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0377 | max_backward: 0.2025 | max_optimizer_step: 0.1246 | max_tensorboard_logging: 0.3552 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4425.2 | uss_MB: 3666.0 | child_rss_MB: 3999.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:45:29,534] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:46:10,648] INFO : Step 20700/50000 rank0 | Train Loss: 8.3612 | Train Loss Unweighted: 2.9612 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6405 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1388 | Regression_eta: 0.0132 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9442 | Classification_binary: 2.6405 | Regression_cos_phi: 0.5144 | Regression_energy: 1.5211 | Regression_eta: 0.6488 | Regression_pt: 1.5973 | Regression_sin_phi: 0.4948 | LR: 6.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6531s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0032 | max_backward: 0.3075 | max_optimizer_step: 0.1246 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4425.2 | uss_MB: 3665.9 | child_rss_MB: 3998.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:46:10,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:46:52,324] INFO : Step 20800/50000 rank0 | Train Loss: 8.1590 | Train Loss Unweighted: 3.0229 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.7080 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1389 | Regression_eta: 0.0118 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8471 | Classification_binary: 2.7080 | Regression_cos_phi: 0.4731 | Regression_energy: 1.5224 | Regression_eta: 0.5805 | Regression_pt: 1.5848 | Regression_sin_phi: 0.4431 | LR: 6.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7314s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0032 | max_backward: 0.3787 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4424.2 | uss_MB: 3666.4 | child_rss_MB: 3999.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:46:52,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:47:34,076] INFO : Step 20900/50000 rank0 | Train Loss: 8.3585 | Train Loss Unweighted: 3.0257 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.7012 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1422 | Regression_eta: 0.0109 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9429 | Classification_binary: 2.7012 | Regression_cos_phi: 0.5138 | Regression_energy: 1.5582 | Regression_eta: 0.5367 | Regression_pt: 1.6349 | Regression_sin_phi: 0.4710 | LR: 6.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6294s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0031 | max_backward: 0.2970 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4424.2 | uss_MB: 3665.5 | child_rss_MB: 4000.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:47:34,291] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:14,554] INFO : Step 21000/50000 rank0 | Train Loss: 8.6959 | Train Loss Unweighted: 2.9541 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6030 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1540 | Regression_eta: 0.0129 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8359 | Classification_binary: 2.6030 | Regression_cos_phi: 0.5367 | Regression_energy: 1.6879 | Regression_eta: 0.6348 | Regression_pt: 1.7844 | Regression_sin_phi: 0.6132 | LR: 6.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5863s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1187 | max_loss: 0.0032 | max_backward: 0.2870 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4425.2 | uss_MB: 3668.0 | child_rss_MB: 4000.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:48:14,774] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:14,795] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 13:48:14,966] INFO : Running validation on rank0 +rank0 [2026-10-28 13:48:14,966] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:48:14,966] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:48:16,282] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 13:48:16,430] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:48:16,563] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:48:16,637] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:48:16,706] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:48:16,775] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:48:16,845] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:48:16,914] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:48:17,026] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:48:17,158] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:48:17,291] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:48:17,362] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:48:17,431] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:48:17,499] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:48:17,570] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:48:17,639] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:48:17,754] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:48:17,762] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:48:17,764] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:48:17,767] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:48:17,770] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:48:17,772] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:48:17,774] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:48:17,777] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:48:17,780] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:48:17,782] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:48:17,783] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:48:17,786] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:48:17,789] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:48:17,791] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:48:17,792] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:48:17,795] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:48:25,485] INFO : VALIDATION | Step=21000/50000 | Train Loss=8.6959 | Valid Loss=8.1990 | Train Loss Unweighted=2.9541 | Valid Loss Unweighted=2.9126 | Stale=0 | ETA=215.2m +rank0 [2026-10-28 13:48:25,495] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:48:25,564] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:48:25,567] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:48:25,691] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:48:27,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:48:28,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:48:29,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:48:29,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:48:29,887] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:48:29,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:48:29,946] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:48:29,948] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:48:30,047] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:48:31,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:48:32,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:48:33,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:48:33,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:48:34,118] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:48:34,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:48:34,175] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:48:34,178] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:48:34,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:48:36,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:48:36,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:48:37,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:48:38,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:48:38,229] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:48:38,236] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:48:38,490] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:48:38,893] INFO : Plotted number of elements +rank0 [2026-10-28 13:48:40,329] INFO : Plotted jets +rank0 [2026-10-28 13:48:41,411] INFO : Plotted jet ratio +rank0 [2026-10-28 13:48:41,446] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:48:41,672] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:48:42,560] INFO : Plotted number of elements +rank0 [2026-10-28 13:48:43,947] INFO : Plotted jets +rank0 [2026-10-28 13:48:45,056] INFO : Plotted jet ratio +rank0 [2026-10-28 13:48:45,090] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 13:48:45,296] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_21000/cld_edm_qq_pf +rank0 [2026-10-28 13:48:45,655] INFO : Plotted number of elements +rank0 [2026-10-28 13:48:47,032] INFO : Plotted jets +rank0 [2026-10-28 13:48:48,098] INFO : Plotted jet ratio +rank0 [2026-10-28 13:49:29,617] INFO : Step 21100/50000 rank0 | Train Loss: 8.4728 | Train Loss Unweighted: 2.9789 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6550 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1400 | Regression_eta: 0.0128 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8869 | Classification_binary: 2.6550 | Regression_cos_phi: 0.6293 | Regression_energy: 1.5344 | Regression_eta: 0.6304 | Regression_pt: 1.6147 | Regression_sin_phi: 0.5222 | LR: 6.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5613s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0582 | max_backward: 0.2023 | max_optimizer_step: 0.1268 | max_tensorboard_logging: 0.3362 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4624.4 | uss_MB: 3865.5 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:49:29,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:49:34,829] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 13:49:34,829] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:50:10,546] INFO : Step 21200/50000 rank0 | Train Loss: 8.4097 | Train Loss Unweighted: 2.9738 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6443 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1448 | Regression_eta: 0.0136 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9466 | Classification_binary: 2.6443 | Regression_cos_phi: 0.4363 | Regression_energy: 1.5869 | Regression_eta: 0.6703 | Regression_pt: 1.6408 | Regression_sin_phi: 0.4846 | LR: 6.56e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5119s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0157 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3000 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4498.1 | uss_MB: 3738.2 | child_rss_MB: 4000.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:50:10,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:50:51,978] INFO : Step 21300/50000 rank0 | Train Loss: 7.7799 | Train Loss Unweighted: 2.8683 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5589 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1365 | Regression_eta: 0.0091 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.5589 | Regression_cos_phi: 0.4313 | Regression_energy: 1.4953 | Regression_eta: 0.4496 | Regression_pt: 1.5932 | Regression_sin_phi: 0.3915 | LR: 6.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6471s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0792 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3091 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.2740 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4498.1 | uss_MB: 3739.2 | child_rss_MB: 4021.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:50:52,194] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:51:33,694] INFO : Step 21400/50000 rank0 | Train Loss: 7.5827 | Train Loss Unweighted: 2.7862 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.4968 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1262 | Regression_eta: 0.0082 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9231 | Classification_binary: 2.4968 | Regression_cos_phi: 0.5027 | Regression_energy: 1.3827 | Regression_eta: 0.4033 | Regression_pt: 1.4553 | Regression_sin_phi: 0.4189 | LR: 6.51e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5741s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0802 | max_backward: 0.2030 | max_optimizer_step: 0.1279 | max_tensorboard_logging: 0.3275 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4498.1 | uss_MB: 3739.5 | child_rss_MB: 3999.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:51:33,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:52:14,262] INFO : Step 21500/50000 rank0 | Train Loss: 7.9749 | Train Loss Unweighted: 2.9532 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6541 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1280 | Regression_eta: 0.0142 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9367 | Classification_binary: 2.6541 | Regression_cos_phi: 0.4368 | Regression_energy: 1.4022 | Regression_eta: 0.6976 | Regression_pt: 1.4993 | Regression_sin_phi: 0.3480 | LR: 6.48e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5894s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.2340 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3798 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4498.1 | uss_MB: 3739.5 | child_rss_MB: 3996.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:52:14,473] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:52:54,910] INFO : Step 21600/50000 rank0 | Train Loss: 7.9957 | Train Loss Unweighted: 2.8753 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.5437 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1471 | Regression_eta: 0.0085 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8874 | Classification_binary: 2.5437 | Regression_cos_phi: 0.4589 | Regression_energy: 1.6114 | Regression_eta: 0.4174 | Regression_pt: 1.7395 | Regression_sin_phi: 0.3374 | LR: 6.45e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5815s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.2304 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4498.1 | uss_MB: 3739.6 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:52:55,122] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:53:35,490] INFO : Step 21700/50000 rank0 | Train Loss: 7.9409 | Train Loss Unweighted: 3.0024 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.6883 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1391 | Regression_eta: 0.0100 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7975 | Classification_binary: 2.6883 | Regression_cos_phi: 0.4199 | Regression_energy: 1.5244 | Regression_eta: 0.4931 | Regression_pt: 1.6272 | Regression_sin_phi: 0.3904 | LR: 6.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6657s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.3087 | max_optimizer_step: 0.1264 | max_tensorboard_logging: 0.3815 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4498.1 | uss_MB: 3739.6 | child_rss_MB: 3995.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:53:35,704] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:54:17,375] INFO : Step 21800/50000 rank0 | Train Loss: 8.1084 | Train Loss Unweighted: 2.9406 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6312 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1332 | Regression_eta: 0.0121 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9257 | Classification_binary: 2.6312 | Regression_cos_phi: 0.5164 | Regression_energy: 1.4596 | Regression_eta: 0.5966 | Regression_pt: 1.5596 | Regression_sin_phi: 0.4191 | LR: 6.40e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5395s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1844 | max_loss: 0.0035 | max_backward: 0.2023 | max_optimizer_step: 0.1231 | max_tensorboard_logging: 0.2928 | max_tensorboard_flush: 0.0013 | Memory: rss_MB: 4496.0 | uss_MB: 3738.2 | child_rss_MB: 3996.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:54:17,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:54:58,321] INFO : Step 21900/50000 rank0 | Train Loss: 8.5814 | Train Loss Unweighted: 2.9783 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6402 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1465 | Regression_eta: 0.0161 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8941 | Classification_binary: 2.6402 | Regression_cos_phi: 0.5331 | Regression_energy: 1.6054 | Regression_eta: 0.7939 | Regression_pt: 1.7039 | Regression_sin_phi: 0.4108 | LR: 6.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6091s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1581 | max_loss: 0.0032 | max_backward: 0.2023 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.2750 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4497.0 | uss_MB: 3739.1 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:54:58,554] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:39,990] INFO : Step 22000/50000 rank0 | Train Loss: 8.4155 | Train Loss Unweighted: 2.9393 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.6007 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1488 | Regression_eta: 0.0114 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8913 | Classification_binary: 2.6007 | Regression_cos_phi: 0.5822 | Regression_energy: 1.6304 | Regression_eta: 0.5629 | Regression_pt: 1.7302 | Regression_sin_phi: 0.4178 | LR: 6.34e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6643s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3106 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3832 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4497.0 | uss_MB: 3739.3 | child_rss_MB: 3998.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:55:40,201] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:40,221] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 13:55:40,512] INFO : Running validation on rank0 +rank0 [2026-10-28 13:55:40,512] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:55:40,512] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:55:41,990] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 13:55:42,136] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:55:42,268] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:55:42,341] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:55:42,410] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:55:42,478] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:55:42,547] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:55:42,616] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:55:42,727] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:55:42,859] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:55:42,990] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:55:43,061] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:55:43,130] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:55:43,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:55:43,268] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:55:43,338] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:55:43,453] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:55:43,460] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:55:43,463] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:55:43,466] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:55:43,468] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:55:43,469] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:55:43,472] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:55:43,475] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:55:43,476] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:55:43,479] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:55:43,481] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:55:43,482] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:55:43,485] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:55:43,486] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:55:43,489] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:55:43,492] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:55:50,922] INFO : VALIDATION | Step=22000/50000 | Train Loss=8.4155 | Valid Loss=8.1094 | Train Loss Unweighted=2.9393 | Valid Loss Unweighted=2.8928 | Stale=0 | ETA=207.8m +rank0 [2026-10-28 13:55:50,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:51,031] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:55:51,034] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 13:55:51,225] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:55:53,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:55:53,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:55:54,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:55:55,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:55:55,352] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:55:55,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:55,429] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:55:55,432] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:55:55,511] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:55:57,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:55:57,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:55:58,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:55:59,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:55:59,301] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:55:59,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:59,369] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 13:55:59,372] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 13:55:59,474] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:56:01,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:56:01,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:56:02,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:56:03,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:56:03,428] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:56:03,452] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:56:03,714] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-10-28 13:56:04,119] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:05,555] INFO : Plotted jets +rank0 [2026-10-28 13:56:06,644] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:06,677] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:56:06,904] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 13:56:07,830] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:09,215] INFO : Plotted jets +rank0 [2026-10-28 13:56:10,319] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:10,351] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 13:56:10,558] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_22000/cld_edm_qq_pf +rank0 [2026-10-28 13:56:10,917] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:12,286] INFO : Plotted jets +rank0 [2026-10-28 13:56:13,339] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:54,797] INFO : Step 22100/50000 rank0 | Train Loss: 7.7992 | Train Loss Unweighted: 2.8871 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5816 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1337 | Regression_eta: 0.0093 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8173 | Classification_binary: 2.5816 | Regression_cos_phi: 0.4777 | Regression_energy: 1.4646 | Regression_eta: 0.4557 | Regression_pt: 1.5755 | Regression_sin_phi: 0.4267 | LR: 6.32e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7047s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3113 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3874 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4693.8 | uss_MB: 3937.2 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:56:55,010] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:57:36,177] INFO : Step 22200/50000 rank0 | Train Loss: 8.1853 | Train Loss Unweighted: 2.9477 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.6306 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1378 | Regression_eta: 0.0109 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8884 | Classification_binary: 2.6306 | Regression_cos_phi: 0.5101 | Regression_energy: 1.5098 | Regression_eta: 0.5346 | Regression_pt: 1.6078 | Regression_sin_phi: 0.5040 | LR: 6.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6613s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0014 | max_forward: 0.0780 | max_loss: 0.0031 | max_backward: 0.2807 | max_optimizer_step: 0.1276 | max_tensorboard_logging: 0.3808 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4693.8 | uss_MB: 3937.3 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:57:36,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:16,992] INFO : Step 22300/50000 rank0 | Train Loss: 8.4874 | Train Loss Unweighted: 2.9227 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.5993 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1427 | Regression_eta: 0.0146 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9553 | Classification_binary: 2.5993 | Regression_cos_phi: 0.4938 | Regression_energy: 1.5635 | Regression_eta: 0.7161 | Regression_pt: 1.5443 | Regression_sin_phi: 0.6151 | LR: 6.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5798s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1842 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4572.2 | uss_MB: 3816.2 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:58:17,209] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:58,234] INFO : Step 22400/50000 rank0 | Train Loss: 8.2947 | Train Loss Unweighted: 2.8942 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.5767 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1371 | Regression_eta: 0.0131 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 1.0467 | Classification_binary: 2.5767 | Regression_cos_phi: 0.4954 | Regression_energy: 1.5026 | Regression_eta: 0.6429 | Regression_pt: 1.5630 | Regression_sin_phi: 0.4674 | LR: 6.23e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6526s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1046 | max_loss: 0.0934 | max_backward: 0.2868 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.2833 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4572.2 | uss_MB: 3818.4 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 13:58:58,447] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:59:39,074] INFO : Step 22500/50000 rank0 | Train Loss: 8.2244 | Train Loss Unweighted: 2.9399 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6272 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1335 | Regression_eta: 0.0149 | Regression_pt: 0.1316 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8365 | Classification_binary: 2.6272 | Regression_cos_phi: 0.5829 | Regression_energy: 1.4624 | Regression_eta: 0.7343 | Regression_pt: 1.5787 | Regression_sin_phi: 0.4025 | LR: 6.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6642s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0034 | max_backward: 0.2025 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4572.2 | uss_MB: 3816.3 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:59:39,300] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:00:20,798] INFO : Step 22600/50000 rank0 | Train Loss: 8.0256 | Train Loss Unweighted: 2.8441 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5339 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1356 | Regression_eta: 0.0137 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8493 | Classification_binary: 2.5339 | Regression_cos_phi: 0.4901 | Regression_energy: 1.4855 | Regression_eta: 0.6715 | Regression_pt: 1.5415 | Regression_sin_phi: 0.4539 | LR: 6.18e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6862s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3096 | max_optimizer_step: 0.1296 | max_tensorboard_logging: 0.3901 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4572.2 | uss_MB: 3816.4 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:00:21,011] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:01,782] INFO : Step 22700/50000 rank0 | Train Loss: 8.4938 | Train Loss Unweighted: 2.9796 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6428 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1470 | Regression_eta: 0.0109 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9236 | Classification_binary: 2.6428 | Regression_cos_phi: 0.5413 | Regression_energy: 1.6103 | Regression_eta: 0.5353 | Regression_pt: 1.7146 | Regression_sin_phi: 0.5260 | LR: 6.15e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6714s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1361 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.3070 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 4572.2 | uss_MB: 3816.5 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:01:01,995] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:44,258] INFO : Step 22800/50000 rank0 | Train Loss: 7.9453 | Train Loss Unweighted: 2.9457 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6396 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1338 | Regression_eta: 0.0115 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8059 | Classification_binary: 2.6396 | Regression_cos_phi: 0.4581 | Regression_energy: 1.4661 | Regression_eta: 0.5635 | Regression_pt: 1.5564 | Regression_sin_phi: 0.4556 | LR: 6.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5691s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2442 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4572.2 | uss_MB: 3816.6 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:01:44,470] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:02:25,656] INFO : Step 22900/50000 rank0 | Train Loss: 8.8741 | Train Loss Unweighted: 2.9670 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6191 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1516 | Regression_eta: 0.0143 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9386 | Classification_binary: 2.6191 | Regression_cos_phi: 0.7624 | Regression_energy: 1.6608 | Regression_eta: 0.7041 | Regression_pt: 1.7213 | Regression_sin_phi: 0.4678 | LR: 6.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5532s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0404 | max_backward: 0.2024 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3394 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4572.1 | uss_MB: 3816.4 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:02:25,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:05,587] INFO : Step 23000/50000 rank0 | Train Loss: 7.5337 | Train Loss Unweighted: 2.7947 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5004 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1294 | Regression_eta: 0.0093 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8350 | Classification_binary: 2.5004 | Regression_cos_phi: 0.3952 | Regression_energy: 1.4174 | Regression_eta: 0.4548 | Regression_pt: 1.5025 | Regression_sin_phi: 0.4285 | LR: 6.06e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7219s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3919 | max_optimizer_step: 0.1277 | max_tensorboard_logging: 0.3436 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4572.1 | uss_MB: 3818.8 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:03:05,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:05,860] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 14:03:06,036] INFO : Running validation on rank0 +rank0 [2026-10-28 14:03:06,036] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:03:06,036] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:03:07,362] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 14:03:07,508] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:03:07,639] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:03:07,713] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:03:07,783] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:03:07,851] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:03:07,921] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:03:07,990] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:03:08,102] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:03:08,233] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:03:08,364] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:03:08,436] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:03:08,505] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:03:08,573] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:03:08,642] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:03:08,712] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:03:08,830] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:03:08,837] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:03:08,839] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:03:08,842] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:03:08,844] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:03:08,846] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:03:08,849] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:03:08,850] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:03:08,852] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:03:08,854] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:03:08,857] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:03:08,859] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:03:08,862] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:03:08,863] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:03:08,865] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:03:08,868] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:03:16,467] INFO : VALIDATION | Step=23000/50000 | Train Loss=7.5337 | Valid Loss=8.0745 | Train Loss Unweighted=2.7947 | Valid Loss Unweighted=2.8788 | Stale=0 | ETA=200.4m +rank0 [2026-10-28 14:03:16,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:03:16,546] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:03:16,550] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:03:16,713] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:03:18,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:03:19,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:03:20,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:03:20,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:03:20,749] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:03:20,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:03:20,808] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:03:20,811] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:03:20,913] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:03:22,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:03:23,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:03:24,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:03:24,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:03:24,787] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:03:24,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:03:24,844] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:03:24,847] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:03:24,950] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:03:26,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:03:27,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:03:28,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:03:28,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:03:28,937] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:03:28,944] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:03:29,209] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:03:29,611] INFO : Plotted number of elements +rank0 [2026-10-28 14:03:31,053] INFO : Plotted jets +rank0 [2026-10-28 14:03:32,657] INFO : Plotted jet ratio +rank0 [2026-10-28 14:03:32,689] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:03:32,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:03:33,312] INFO : Plotted number of elements +rank0 [2026-10-28 14:03:34,704] INFO : Plotted jets +rank0 [2026-10-28 14:03:35,819] INFO : Plotted jet ratio +rank0 [2026-10-28 14:03:35,850] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 14:03:36,057] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_23000/cld_edm_qq_pf +rank0 [2026-10-28 14:03:36,419] INFO : Plotted number of elements +rank0 [2026-10-28 14:03:37,796] INFO : Plotted jets +rank0 [2026-10-28 14:03:38,856] INFO : Plotted jet ratio +rank0 [2026-10-28 14:04:20,498] INFO : Step 23100/50000 rank0 | Train Loss: 8.8554 | Train Loss Unweighted: 3.0243 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.6825 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1455 | Regression_eta: 0.0186 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8274 | Classification_binary: 2.6825 | Regression_cos_phi: 0.6238 | Regression_energy: 1.5943 | Regression_eta: 0.9159 | Regression_pt: 1.7167 | Regression_sin_phi: 0.4948 | LR: 6.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6204s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2285 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.3880 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4756.7 | uss_MB: 4002.5 | child_rss_MB: 3998.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:04:20,712] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:00,737] INFO : Step 23200/50000 rank0 | Train Loss: 8.1110 | Train Loss Unweighted: 2.9762 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6645 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1355 | Regression_eta: 0.0101 | Regression_pt: 0.1316 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9519 | Classification_binary: 2.6645 | Regression_cos_phi: 0.5202 | Regression_energy: 1.4845 | Regression_eta: 0.4951 | Regression_pt: 1.5786 | Regression_sin_phi: 0.4161 | LR: 6.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1860 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.1321 | max_tensorboard_logging: 0.2906 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4756.7 | uss_MB: 4002.5 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:05:00,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:41,710] INFO : Step 23300/50000 rank0 | Train Loss: 7.5451 | Train Loss Unweighted: 2.8679 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5676 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1338 | Regression_eta: 0.0063 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8359 | Classification_binary: 2.5676 | Regression_cos_phi: 0.3915 | Regression_energy: 1.4658 | Regression_eta: 0.3093 | Regression_pt: 1.5595 | Regression_sin_phi: 0.4155 | LR: 5.98e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.6635s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3101 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3718 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4641.3 | uss_MB: 3887.3 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:05:41,924] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:06:22,175] INFO : Step 23400/50000 rank0 | Train Loss: 7.7281 | Train Loss Unweighted: 2.8641 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5596 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1325 | Regression_eta: 0.0089 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8765 | Classification_binary: 2.5596 | Regression_cos_phi: 0.4498 | Regression_energy: 1.4513 | Regression_eta: 0.4379 | Regression_pt: 1.5832 | Regression_sin_phi: 0.3698 | LR: 5.95e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7471s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1492 | max_loss: 0.0035 | max_backward: 0.3947 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4642.3 | uss_MB: 3889.4 | child_rss_MB: 4019.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:06:22,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:07:03,358] INFO : Step 23500/50000 rank0 | Train Loss: 7.8263 | Train Loss Unweighted: 2.9497 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6423 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1349 | Regression_eta: 0.0084 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9228 | Classification_binary: 2.6423 | Regression_cos_phi: 0.3874 | Regression_energy: 1.4783 | Regression_eta: 0.4111 | Regression_pt: 1.5879 | Regression_sin_phi: 0.3966 | LR: 5.92e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1787 | max_loss: 0.0035 | max_backward: 0.2028 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4642.3 | uss_MB: 3887.4 | child_rss_MB: 3996.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:07:03,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:07:46,001] INFO : Step 23600/50000 rank0 | Train Loss: 7.7871 | Train Loss Unweighted: 2.7430 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4451 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1294 | Regression_eta: 0.0142 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8649 | Classification_binary: 2.4451 | Regression_cos_phi: 0.4941 | Regression_energy: 1.4177 | Regression_eta: 0.6976 | Regression_pt: 1.4667 | Regression_sin_phi: 0.4010 | LR: 5.90e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6579s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1828 | max_loss: 0.0037 | max_backward: 0.2026 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.2771 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4642.3 | uss_MB: 3887.4 | child_rss_MB: 4000.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:07:46,227] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:08:27,765] INFO : Step 23700/50000 rank0 | Train Loss: 7.9213 | Train Loss Unweighted: 2.7989 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4817 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1376 | Regression_eta: 0.0121 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8508 | Classification_binary: 2.4817 | Regression_cos_phi: 0.4284 | Regression_energy: 1.5080 | Regression_eta: 0.5965 | Regression_pt: 1.6346 | Regression_sin_phi: 0.4212 | LR: 5.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7205s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1497 | max_loss: 0.0037 | max_backward: 0.3936 | max_optimizer_step: 0.1308 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4642.3 | uss_MB: 3889.5 | child_rss_MB: 4000.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 14:08:27,979] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:08,925] INFO : Step 23800/50000 rank0 | Train Loss: 8.6021 | Train Loss Unweighted: 2.8929 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5507 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1486 | Regression_eta: 0.0183 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8738 | Classification_binary: 2.5507 | Regression_cos_phi: 0.4889 | Regression_energy: 1.6285 | Regression_eta: 0.8974 | Regression_pt: 1.7073 | Regression_sin_phi: 0.4556 | LR: 5.84e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.5796s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1808 | max_loss: 0.0036 | max_backward: 0.2029 | max_optimizer_step: 0.1280 | max_tensorboard_logging: 0.2863 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4642.3 | uss_MB: 3887.3 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:09:09,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:50,059] INFO : Step 23900/50000 rank0 | Train Loss: 7.9644 | Train Loss Unweighted: 2.9233 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.6122 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1346 | Regression_eta: 0.0104 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8771 | Classification_binary: 2.6122 | Regression_cos_phi: 0.4395 | Regression_energy: 1.4752 | Regression_eta: 0.5120 | Regression_pt: 1.6059 | Regression_sin_phi: 0.4425 | LR: 5.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6693s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0032 | max_backward: 0.3107 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4641.8 | uss_MB: 3887.2 | child_rss_MB: 3999.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:09:50,270] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:10:31,697] INFO : Step 24000/50000 rank0 | Train Loss: 8.4389 | Train Loss Unweighted: 2.8459 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5152 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1424 | Regression_eta: 0.0132 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9725 | Classification_binary: 2.5152 | Regression_cos_phi: 0.5393 | Regression_energy: 1.5602 | Regression_eta: 0.6468 | Regression_pt: 1.6517 | Regression_sin_phi: 0.5530 | LR: 5.78e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5028s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0294 | max_forward: 0.0782 | max_loss: 0.0030 | max_backward: 0.2026 | max_optimizer_step: 0.1506 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4641.8 | uss_MB: 3887.3 | child_rss_MB: 3995.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:10:31,925] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:10:31,944] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 14:10:32,120] INFO : Running validation on rank0 +rank0 [2026-10-28 14:10:32,121] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:10:32,121] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:10:33,479] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 14:10:33,624] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:10:33,757] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:10:33,832] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:10:33,902] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:10:33,971] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:10:34,039] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:10:34,109] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:10:34,221] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:10:34,353] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:10:34,485] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:10:34,556] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:10:34,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:10:34,694] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:10:34,763] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:10:34,833] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:10:34,947] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:10:34,955] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:10:34,958] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:10:34,961] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:10:34,962] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:10:34,965] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:10:34,967] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:10:34,968] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:10:34,971] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:10:34,974] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:10:34,977] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:10:34,980] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:10:34,981] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:10:34,983] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:10:34,984] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:10:34,988] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:10:42,680] INFO : VALIDATION | Step=24000/50000 | Train Loss=8.4389 | Valid Loss=8.0030 | Train Loss Unweighted=2.8459 | Valid Loss Unweighted=2.8488 | Stale=0 | ETA=193.0m +rank0 [2026-10-28 14:10:42,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:10:42,790] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:10:42,794] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:10:42,933] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:10:45,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:10:45,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:10:46,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:10:47,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:10:47,235] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:10:47,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:10:47,322] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:10:47,325] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:10:47,399] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:10:49,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:10:49,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:10:50,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:10:51,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:10:51,347] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:10:51,350] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:10:51,417] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:10:51,420] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:10:51,507] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:10:53,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:10:54,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:10:54,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:10:55,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:10:55,510] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:10:55,517] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:10:55,779] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:10:56,183] INFO : Plotted number of elements +rank0 [2026-10-28 14:10:57,623] INFO : Plotted jets +rank0 [2026-10-28 14:10:59,221] INFO : Plotted jet ratio +rank0 [2026-10-28 14:10:59,255] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:10:59,484] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:10:59,872] INFO : Plotted number of elements +rank0 [2026-10-28 14:11:01,268] INFO : Plotted jets +rank0 [2026-10-28 14:11:02,382] INFO : Plotted jet ratio +rank0 [2026-10-28 14:11:02,436] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 14:11:02,641] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_24000/cld_edm_qq_pf +rank0 [2026-10-28 14:11:03,004] INFO : Plotted number of elements +rank0 [2026-10-28 14:11:04,391] INFO : Plotted jets +rank0 [2026-10-28 14:11:05,451] INFO : Plotted jet ratio +rank0 [2026-10-28 14:11:46,396] INFO : Step 24100/50000 rank0 | Train Loss: 8.6500 | Train Loss Unweighted: 3.0110 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.6533 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1577 | Regression_eta: 0.0087 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8764 | Classification_binary: 2.6533 | Regression_cos_phi: 0.6090 | Regression_energy: 1.7277 | Regression_eta: 0.4299 | Regression_pt: 1.8715 | Regression_sin_phi: 0.4821 | LR: 5.75e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6891s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3107 | max_optimizer_step: 0.1360 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4853.3 | uss_MB: 4097.4 | child_rss_MB: 3995.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:11:46,610] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:12:27,174] INFO : Step 24200/50000 rank0 | Train Loss: 8.6277 | Train Loss Unweighted: 2.9879 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.6362 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1549 | Regression_eta: 0.0119 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8555 | Classification_binary: 2.6362 | Regression_cos_phi: 0.6247 | Regression_energy: 1.6972 | Regression_eta: 0.5850 | Regression_pt: 1.8091 | Regression_sin_phi: 0.4199 | LR: 5.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6567s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.1074 | max_backward: 0.2024 | max_optimizer_step: 0.1319 | max_tensorboard_logging: 0.3219 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4853.3 | uss_MB: 4097.4 | child_rss_MB: 3996.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:12:27,385] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:08,745] INFO : Step 24300/50000 rank0 | Train Loss: 8.3283 | Train Loss Unweighted: 2.9707 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.6434 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1436 | Regression_eta: 0.0123 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8132 | Classification_binary: 2.6434 | Regression_cos_phi: 0.4931 | Regression_energy: 1.5735 | Regression_eta: 0.6036 | Regression_pt: 1.6588 | Regression_sin_phi: 0.5427 | LR: 5.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5311s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0706 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2026 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.2728 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4717.9 | uss_MB: 3961.6 | child_rss_MB: 3997.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:13:08,979] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:50,097] INFO : Step 24400/50000 rank0 | Train Loss: 8.3115 | Train Loss Unweighted: 2.8676 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5466 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1377 | Regression_eta: 0.0178 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9033 | Classification_binary: 2.5466 | Regression_cos_phi: 0.4707 | Regression_energy: 1.5092 | Regression_eta: 0.8735 | Regression_pt: 1.5931 | Regression_sin_phi: 0.4151 | LR: 5.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6907s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1846 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4717.9 | uss_MB: 3961.7 | child_rss_MB: 3999.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:13:50,325] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:14:32,439] INFO : Step 24500/50000 rank0 | Train Loss: 8.4930 | Train Loss Unweighted: 2.8531 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5234 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1426 | Regression_eta: 0.0137 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9471 | Classification_binary: 2.5234 | Regression_cos_phi: 0.6283 | Regression_energy: 1.5625 | Regression_eta: 0.6729 | Regression_pt: 1.6277 | Regression_sin_phi: 0.5312 | LR: 5.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6986s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3095 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.3829 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4717.9 | uss_MB: 3961.8 | child_rss_MB: 4001.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:14:32,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:15:13,182] INFO : Step 24600/50000 rank0 | Train Loss: 8.1257 | Train Loss Unweighted: 2.9536 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6371 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1365 | Regression_eta: 0.0109 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8797 | Classification_binary: 2.6371 | Regression_cos_phi: 0.4885 | Regression_energy: 1.4955 | Regression_eta: 0.5368 | Regression_pt: 1.6301 | Regression_sin_phi: 0.4579 | LR: 5.61e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7521s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1040 | max_loss: 0.0033 | max_backward: 0.3919 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.3547 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4717.9 | uss_MB: 3963.9 | child_rss_MB: 3999.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 14:15:13,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:15:55,184] INFO : Step 24700/50000 rank0 | Train Loss: 8.7131 | Train Loss Unweighted: 2.8448 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4946 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1524 | Regression_eta: 0.0133 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9652 | Classification_binary: 2.4946 | Regression_cos_phi: 0.6179 | Regression_energy: 1.6703 | Regression_eta: 0.6538 | Regression_pt: 1.7513 | Regression_sin_phi: 0.5599 | LR: 5.58e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8573s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0487 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.3965 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4717.9 | uss_MB: 3963.9 | child_rss_MB: 3997.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:15:55,399] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:16:36,607] INFO : Step 24800/50000 rank0 | Train Loss: 8.1072 | Train Loss Unweighted: 2.8987 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5808 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1395 | Regression_eta: 0.0084 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9021 | Classification_binary: 2.5808 | Regression_cos_phi: 0.5141 | Regression_energy: 1.5280 | Regression_eta: 0.4117 | Regression_pt: 1.6126 | Regression_sin_phi: 0.5581 | LR: 5.56e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5466s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0350 | max_backward: 0.2032 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3309 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4717.9 | uss_MB: 3961.9 | child_rss_MB: 4001.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:16:36,819] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:17,899] INFO : Step 24900/50000 rank0 | Train Loss: 8.1657 | Train Loss Unweighted: 2.9159 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.5940 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1405 | Regression_eta: 0.0134 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7835 | Classification_binary: 2.5940 | Regression_cos_phi: 0.4851 | Regression_energy: 1.5398 | Regression_eta: 0.6604 | Regression_pt: 1.6414 | Regression_sin_phi: 0.4613 | LR: 5.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6641s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.2927 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.3846 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4718.5 | uss_MB: 3961.6 | child_rss_MB: 4001.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:17:18,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:58,814] INFO : Step 25000/50000 rank0 | Train Loss: 8.3112 | Train Loss Unweighted: 2.9163 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.5844 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1470 | Regression_eta: 0.0133 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7939 | Classification_binary: 2.5844 | Regression_cos_phi: 0.5939 | Regression_energy: 1.6102 | Regression_eta: 0.6555 | Regression_pt: 1.6768 | Regression_sin_phi: 0.3965 | LR: 5.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5832s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0918 | max_loss: 0.0034 | max_backward: 0.2028 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4718.5 | uss_MB: 3961.7 | child_rss_MB: 4004.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:17:59,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:59,060] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 14:17:59,235] INFO : Running validation on rank0 +rank0 [2026-10-28 14:17:59,235] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:17:59,235] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:18:00,588] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 14:18:00,732] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:18:00,865] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:18:00,940] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:18:01,009] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:18:01,077] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:18:01,146] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:18:01,216] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:18:01,328] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:18:01,460] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:18:01,592] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:18:01,665] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:18:01,734] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:18:01,802] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:18:01,871] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:18:01,941] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:18:02,053] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:18:02,060] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:18:02,063] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:18:02,064] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:18:02,067] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:18:02,070] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:18:02,072] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:18:02,074] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:18:02,078] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:18:02,081] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:18:02,083] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:18:02,086] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:18:02,090] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:18:02,091] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:18:02,094] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:18:02,096] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:18:09,649] INFO : VALIDATION | Step=25000/50000 | Train Loss=8.3112 | Valid Loss=7.9802 | Train Loss Unweighted=2.9163 | Valid Loss Unweighted=2.8465 | Stale=0 | ETA=185.6m +rank0 [2026-10-28 14:18:09,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:18:09,716] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:18:09,719] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:18:09,881] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:18:11,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:18:12,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:18:13,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:18:13,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:18:14,125] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:18:14,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:18:14,186] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:18:14,189] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:18:14,289] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:18:16,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:18:16,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:18:17,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:18:18,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:18:18,370] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:18:18,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:18:18,439] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:18:18,442] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:18:18,533] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:18:20,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:18:21,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:18:21,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:18:22,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:18:22,495] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:18:22,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:18:22,761] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:18:23,166] INFO : Plotted number of elements +rank0 [2026-10-28 14:18:24,603] INFO : Plotted jets +rank0 [2026-10-28 14:18:26,208] INFO : Plotted jet ratio +rank0 [2026-10-28 14:18:26,239] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:18:26,469] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:18:26,856] INFO : Plotted number of elements +rank0 [2026-10-28 14:18:28,242] INFO : Plotted jets +rank0 [2026-10-28 14:18:29,356] INFO : Plotted jet ratio +rank0 [2026-10-28 14:18:29,388] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 14:18:29,592] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_25000/cld_edm_qq_pf +rank0 [2026-10-28 14:18:29,955] INFO : Plotted number of elements +rank0 [2026-10-28 14:18:31,331] INFO : Plotted jets +rank0 [2026-10-28 14:18:32,391] INFO : Plotted jet ratio +rank0 [2026-10-28 14:19:13,749] INFO : Step 25100/50000 rank0 | Train Loss: 7.7500 | Train Loss Unweighted: 2.7268 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4317 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1255 | Regression_eta: 0.0127 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9443 | Classification_binary: 2.4317 | Regression_cos_phi: 0.4815 | Regression_energy: 1.3746 | Regression_eta: 0.6224 | Regression_pt: 1.4766 | Regression_sin_phi: 0.4188 | LR: 5.47e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0124 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.3037 | max_optimizer_step: 0.1324 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4926.6 | uss_MB: 4169.8 | child_rss_MB: 4019.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:19:13,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:19:54,916] INFO : Step 25200/50000 rank0 | Train Loss: 7.7665 | Train Loss Unweighted: 2.8899 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5899 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1320 | Regression_eta: 0.0095 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7971 | Classification_binary: 2.5899 | Regression_cos_phi: 0.5207 | Regression_energy: 1.4465 | Regression_eta: 0.4683 | Regression_pt: 1.5263 | Regression_sin_phi: 0.4178 | LR: 5.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1047 | max_loss: 0.0373 | max_backward: 0.2868 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4926.6 | uss_MB: 4171.9 | child_rss_MB: 4002.7 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 14:19:55,135] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:20:36,937] INFO : Step 25300/50000 rank0 | Train Loss: 8.1388 | Train Loss Unweighted: 2.7887 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4558 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1469 | Regression_eta: 0.0114 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8217 | Classification_binary: 2.4558 | Regression_cos_phi: 0.5204 | Regression_energy: 1.6099 | Regression_eta: 0.5617 | Regression_pt: 1.7031 | Regression_sin_phi: 0.4662 | LR: 5.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0782 | max_loss: 0.0034 | max_backward: 0.2389 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.4007 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 4797.4 | uss_MB: 4038.4 | child_rss_MB: 4001.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:20:37,152] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:21:17,469] INFO : Step 25400/50000 rank0 | Train Loss: 7.5521 | Train Loss Unweighted: 2.8209 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5299 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1296 | Regression_eta: 0.0091 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8202 | Classification_binary: 2.5299 | Regression_cos_phi: 0.4246 | Regression_energy: 1.4196 | Regression_eta: 0.4462 | Regression_pt: 1.4572 | Regression_sin_phi: 0.4545 | LR: 5.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6577s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0413 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3107 | max_optimizer_step: 0.1362 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4797.4 | uss_MB: 4038.6 | child_rss_MB: 4026.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:21:17,682] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:21:58,893] INFO : Step 25500/50000 rank0 | Train Loss: 8.2759 | Train Loss Unweighted: 2.8810 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5547 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1397 | Regression_eta: 0.0134 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9174 | Classification_binary: 2.5547 | Regression_cos_phi: 0.5369 | Regression_energy: 1.5307 | Regression_eta: 0.6612 | Regression_pt: 1.6713 | Regression_sin_phi: 0.4038 | LR: 5.36e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6767s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1824 | max_loss: 0.0037 | max_backward: 0.2039 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 4004.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:21:59,128] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:22:41,407] INFO : Step 25600/50000 rank0 | Train Loss: 7.4296 | Train Loss Unweighted: 2.7492 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4720 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1208 | Regression_eta: 0.0110 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8064 | Classification_binary: 2.4720 | Regression_cos_phi: 0.5656 | Regression_energy: 1.3237 | Regression_eta: 0.5393 | Regression_pt: 1.3721 | Regression_sin_phi: 0.3504 | LR: 5.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.3106 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 4001.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:22:41,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:23:22,466] INFO : Step 25700/50000 rank0 | Train Loss: 8.5130 | Train Loss Unweighted: 2.8451 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5224 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1384 | Regression_eta: 0.0230 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8254 | Classification_binary: 2.5224 | Regression_cos_phi: 0.5394 | Regression_energy: 1.5165 | Regression_eta: 1.1292 | Regression_pt: 1.5481 | Regression_sin_phi: 0.4321 | LR: 5.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6909s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1087 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.3079 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 4026.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:23:22,681] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:03,306] INFO : Step 25800/50000 rank0 | Train Loss: 7.6826 | Train Loss Unweighted: 2.8629 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5724 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1258 | Regression_eta: 0.0107 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8090 | Classification_binary: 2.5724 | Regression_cos_phi: 0.4588 | Regression_energy: 1.3784 | Regression_eta: 0.5268 | Regression_pt: 1.4719 | Regression_sin_phi: 0.4654 | LR: 5.27e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6774s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.3069 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4797.4 | uss_MB: 4038.8 | child_rss_MB: 3999.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:24:03,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:44,067] INFO : Step 25900/50000 rank0 | Train Loss: 8.3263 | Train Loss Unweighted: 2.9525 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.6273 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1417 | Regression_eta: 0.0109 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8502 | Classification_binary: 2.6273 | Regression_cos_phi: 0.6242 | Regression_energy: 1.5529 | Regression_eta: 0.5376 | Regression_pt: 1.6518 | Regression_sin_phi: 0.4823 | LR: 5.25e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5964s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3098 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4797.2 | uss_MB: 4038.5 | child_rss_MB: 4025.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:24:44,280] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:25:25,756] INFO : Step 26000/50000 rank0 | Train Loss: 7.7888 | Train Loss Unweighted: 2.7574 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4545 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1335 | Regression_eta: 0.0106 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9585 | Classification_binary: 2.4545 | Regression_cos_phi: 0.4818 | Regression_energy: 1.4629 | Regression_eta: 0.5222 | Regression_pt: 1.4957 | Regression_sin_phi: 0.4131 | LR: 5.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6045s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0890 | max_backward: 0.2030 | max_optimizer_step: 0.1280 | max_tensorboard_logging: 0.3376 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4797.2 | uss_MB: 4038.5 | child_rss_MB: 4005.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:25:25,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:25:26,004] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 14:25:26,181] INFO : Running validation on rank0 +rank0 [2026-10-28 14:25:26,181] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:25:26,181] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:25:27,567] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 14:25:27,712] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:25:27,845] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:25:27,919] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:25:27,988] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:25:28,057] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:25:28,126] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:25:28,197] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:25:28,310] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:25:28,443] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:25:28,574] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:25:28,645] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:25:28,715] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:25:28,784] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:25:28,854] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:25:28,925] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:25:29,043] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:25:29,050] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:25:29,054] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:25:29,058] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:25:29,062] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:25:29,065] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:25:29,069] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:25:29,073] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:25:29,074] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:25:29,078] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:25:29,081] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:25:29,085] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:25:29,086] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:25:29,090] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:25:29,094] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:25:29,098] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:25:36,794] INFO : VALIDATION | Step=26000/50000 | Train Loss=7.7888 | Valid Loss=7.8593 | Train Loss Unweighted=2.7574 | Valid Loss Unweighted=2.8014 | Stale=0 | ETA=178.2m +rank0 [2026-10-28 14:25:36,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:25:36,878] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:25:36,881] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:25:37,045] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:25:39,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:25:39,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:25:40,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:25:41,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:25:41,450] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:25:41,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:25:41,518] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:25:41,520] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:25:41,610] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:25:43,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:25:44,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:25:44,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:25:45,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:25:45,666] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:25:45,669] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:25:45,748] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:25:45,751] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:25:45,827] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:25:47,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:25:48,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:25:49,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:25:49,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:25:49,830] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:25:49,838] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:25:50,098] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:25:50,506] INFO : Plotted number of elements +rank0 [2026-10-28 14:25:51,951] INFO : Plotted jets +rank0 [2026-10-28 14:25:53,040] INFO : Plotted jet ratio +rank0 [2026-10-28 14:25:53,076] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:25:53,305] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:25:53,692] INFO : Plotted number of elements +rank0 [2026-10-28 14:25:55,653] INFO : Plotted jets +rank0 [2026-10-28 14:25:56,773] INFO : Plotted jet ratio +rank0 [2026-10-28 14:25:56,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 14:25:57,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_26000/cld_edm_qq_pf +rank0 [2026-10-28 14:25:57,381] INFO : Plotted number of elements +rank0 [2026-10-28 14:25:58,762] INFO : Plotted jets +rank0 [2026-10-28 14:25:59,835] INFO : Plotted jet ratio +rank0 [2026-10-28 14:26:40,891] INFO : Step 26100/50000 rank0 | Train Loss: 7.8285 | Train Loss Unweighted: 2.8849 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.5815 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1317 | Regression_eta: 0.0106 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8372 | Classification_binary: 2.5815 | Regression_cos_phi: 0.4195 | Regression_energy: 1.4431 | Regression_eta: 0.5228 | Regression_pt: 1.5550 | Regression_sin_phi: 0.4695 | LR: 5.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6251s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0798 | max_backward: 0.2023 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3196 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4985.1 | uss_MB: 4227.7 | child_rss_MB: 4005.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:26:41,104] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:27:23,306] INFO : Step 26200/50000 rank0 | Train Loss: 7.6889 | Train Loss Unweighted: 2.7059 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3885 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1403 | Regression_eta: 0.0089 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7751 | Classification_binary: 2.3885 | Regression_cos_phi: 0.4902 | Regression_energy: 1.5377 | Regression_eta: 0.4383 | Regression_pt: 1.6573 | Regression_sin_phi: 0.4018 | LR: 5.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7833s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0019 | max_forward: 0.1578 | max_loss: 0.0034 | max_backward: 0.3937 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0138 | Memory: rss_MB: 4985.1 | uss_MB: 4229.8 | child_rss_MB: 4030.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:27:23,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:28:04,576] INFO : Step 26300/50000 rank0 | Train Loss: 8.3302 | Train Loss Unweighted: 2.8500 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.5154 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1472 | Regression_eta: 0.0148 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8001 | Classification_binary: 2.5154 | Regression_cos_phi: 0.5347 | Regression_energy: 1.6124 | Regression_eta: 0.7280 | Regression_pt: 1.6853 | Regression_sin_phi: 0.4541 | LR: 5.13e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6766s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1270 | max_loss: 0.0032 | max_backward: 0.2025 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.3118 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4870.9 | uss_MB: 4111.3 | child_rss_MB: 3996.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:28:04,790] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:28:45,553] INFO : Step 26400/50000 rank0 | Train Loss: 7.3076 | Train Loss Unweighted: 2.7554 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4706 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1241 | Regression_eta: 0.0088 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8672 | Classification_binary: 2.4706 | Regression_cos_phi: 0.3752 | Regression_energy: 1.3601 | Regression_eta: 0.4313 | Regression_pt: 1.4707 | Regression_sin_phi: 0.3325 | LR: 5.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7017s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3056 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3864 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4872.9 | uss_MB: 4112.6 | child_rss_MB: 3999.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:28:45,767] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:29:26,914] INFO : Step 26500/50000 rank0 | Train Loss: 8.3463 | Train Loss Unweighted: 2.8418 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.5301 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1325 | Regression_eta: 0.0209 | Regression_pt: 0.1255 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8170 | Classification_binary: 2.5301 | Regression_cos_phi: 0.5296 | Regression_energy: 1.4513 | Regression_eta: 1.0254 | Regression_pt: 1.5048 | Regression_sin_phi: 0.4881 | LR: 5.08e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8906s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0831 | max_forward: 0.1044 | max_loss: 0.0033 | max_backward: 0.3932 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.3809 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4872.9 | uss_MB: 4114.9 | child_rss_MB: 3995.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:29:27,128] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:07,343] INFO : Step 26600/50000 rank0 | Train Loss: 8.0200 | Train Loss Unweighted: 2.8519 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5497 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1307 | Regression_eta: 0.0135 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8470 | Classification_binary: 2.5497 | Regression_cos_phi: 0.5643 | Regression_energy: 1.4316 | Regression_eta: 0.6647 | Regression_pt: 1.4899 | Regression_sin_phi: 0.4728 | LR: 5.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6870s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0646 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3111 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.2993 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4872.9 | uss_MB: 4112.8 | child_rss_MB: 4027.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:07,554] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:49,941] INFO : Step 26700/50000 rank0 | Train Loss: 8.8658 | Train Loss Unweighted: 2.9904 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6410 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1532 | Regression_eta: 0.0182 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9218 | Classification_binary: 2.6410 | Regression_cos_phi: 0.5631 | Regression_energy: 1.6784 | Regression_eta: 0.8952 | Regression_pt: 1.7145 | Regression_sin_phi: 0.4518 | LR: 5.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5895s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0015 | max_forward: 0.1833 | max_loss: 0.0035 | max_backward: 0.2025 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4872.9 | uss_MB: 4112.9 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:50,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:31:30,390] INFO : Step 26800/50000 rank0 | Train Loss: 7.2546 | Train Loss Unweighted: 2.7137 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.4331 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1241 | Regression_eta: 0.0099 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7953 | Classification_binary: 2.4331 | Regression_cos_phi: 0.3894 | Regression_energy: 1.3595 | Regression_eta: 0.4865 | Regression_pt: 1.4156 | Regression_sin_phi: 0.3751 | LR: 4.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6436s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.2627 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3979 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4872.9 | uss_MB: 4112.9 | child_rss_MB: 3999.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:31:30,602] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:32:12,105] INFO : Step 26900/50000 rank0 | Train Loss: 8.4342 | Train Loss Unweighted: 2.8619 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5072 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1556 | Regression_eta: 0.0153 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9041 | Classification_binary: 2.5072 | Regression_cos_phi: 0.3900 | Regression_energy: 1.7050 | Regression_eta: 0.7519 | Regression_pt: 1.8410 | Regression_sin_phi: 0.3351 | LR: 4.96e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5830s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0961 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4870.7 | uss_MB: 4111.2 | child_rss_MB: 4000.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:32:12,337] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:53,950] INFO : Step 27000/50000 rank0 | Train Loss: 7.7063 | Train Loss Unweighted: 2.7858 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4860 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1306 | Regression_eta: 0.0095 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8215 | Classification_binary: 2.4860 | Regression_cos_phi: 0.5685 | Regression_energy: 1.4305 | Regression_eta: 0.4660 | Regression_pt: 1.5290 | Regression_sin_phi: 0.4049 | LR: 4.94e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2282 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4870.7 | uss_MB: 4112.5 | child_rss_MB: 3998.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:32:54,164] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:54,260] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 14:32:54,434] INFO : Running validation on rank0 +rank0 [2026-10-28 14:32:54,435] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:32:54,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:32:55,757] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 14:32:55,904] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:32:56,037] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:32:56,110] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:32:56,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:32:56,249] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:32:56,320] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:32:56,389] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:32:56,502] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:32:56,635] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:32:56,767] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:32:56,838] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:32:56,908] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:32:56,977] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:32:57,047] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:32:57,117] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:32:57,232] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:32:57,238] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:32:57,241] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:32:57,242] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:32:57,246] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:32:57,248] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:32:57,249] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:32:57,252] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:32:57,255] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:32:57,258] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:32:57,259] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:32:57,262] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:32:57,263] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:32:57,265] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:32:57,266] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:32:57,269] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:33:04,897] INFO : VALIDATION | Step=27000/50000 | Train Loss=7.7063 | Valid Loss=7.9091 | Train Loss Unweighted=2.7858 | Valid Loss Unweighted=2.7975 | Stale=1 | ETA=170.8m +rank0 [2026-10-28 14:33:04,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:04,975] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:33:04,978] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:33:05,145] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:33:07,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:33:07,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:33:08,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:33:08,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:33:09,138] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:33:09,142] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:09,209] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:33:09,212] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:33:09,304] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:33:11,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:33:11,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:33:12,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:33:13,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:33:13,223] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:33:13,226] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:13,290] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:33:13,293] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:33:13,386] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:33:15,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:33:15,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:33:16,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:33:17,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:33:17,361] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:33:17,369] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:33:17,624] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:33:18,026] INFO : Plotted number of elements +rank0 [2026-10-28 14:33:19,468] INFO : Plotted jets +rank0 [2026-10-28 14:33:21,079] INFO : Plotted jet ratio +rank0 [2026-10-28 14:33:21,112] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:33:21,348] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:33:21,739] INFO : Plotted number of elements +rank0 [2026-10-28 14:33:23,146] INFO : Plotted jets +rank0 [2026-10-28 14:33:24,268] INFO : Plotted jet ratio +rank0 [2026-10-28 14:33:24,301] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 14:33:24,506] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_27000/cld_edm_qq_pf +rank0 [2026-10-28 14:33:24,872] INFO : Plotted number of elements +rank0 [2026-10-28 14:33:26,256] INFO : Plotted jets +rank0 [2026-10-28 14:33:27,311] INFO : Plotted jet ratio +rank0 [2026-10-28 14:34:09,011] INFO : Step 27100/50000 rank0 | Train Loss: 7.6648 | Train Loss Unweighted: 2.7359 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4406 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1276 | Regression_eta: 0.0106 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8415 | Classification_binary: 2.4406 | Regression_cos_phi: 0.4980 | Regression_energy: 1.3984 | Regression_eta: 0.5235 | Regression_pt: 1.4916 | Regression_sin_phi: 0.4712 | LR: 4.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7228s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3058 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3896 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5060.2 | uss_MB: 4302.0 | child_rss_MB: 3999.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:09,222] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:50,511] INFO : Step 27200/50000 rank0 | Train Loss: 7.5980 | Train Loss Unweighted: 2.7749 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4839 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1261 | Regression_eta: 0.0106 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7811 | Classification_binary: 2.4839 | Regression_cos_phi: 0.5024 | Regression_energy: 1.3815 | Regression_eta: 0.5195 | Regression_pt: 1.4779 | Regression_sin_phi: 0.4519 | LR: 4.88e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.5562s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0263 | max_backward: 0.2027 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3213 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5060.2 | uss_MB: 4301.9 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:50,723] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:35:31,204] INFO : Step 27300/50000 rank0 | Train Loss: 8.0099 | Train Loss Unweighted: 2.9050 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5891 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1397 | Regression_eta: 0.0153 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8067 | Classification_binary: 2.5891 | Regression_cos_phi: 0.4164 | Regression_energy: 1.5309 | Regression_eta: 0.7536 | Regression_pt: 1.5881 | Regression_sin_phi: 0.3251 | LR: 4.85e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5637s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0197 | max_backward: 0.2025 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3480 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4942.4 | uss_MB: 4184.0 | child_rss_MB: 4001.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:35:31,417] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:36:13,292] INFO : Step 27400/50000 rank0 | Train Loss: 7.5353 | Train Loss Unweighted: 2.7477 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4606 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1243 | Regression_eta: 0.0097 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8770 | Classification_binary: 2.4606 | Regression_cos_phi: 0.5107 | Regression_energy: 1.3616 | Regression_eta: 0.4754 | Regression_pt: 1.4476 | Regression_sin_phi: 0.4025 | LR: 4.82e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6135s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2118 | max_optimizer_step: 0.1369 | max_tensorboard_logging: 0.3793 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4944.4 | uss_MB: 4185.3 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:36:13,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:36:54,273] INFO : Step 27500/50000 rank0 | Train Loss: 8.0608 | Train Loss Unweighted: 2.8422 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5451 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1261 | Regression_eta: 0.0141 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8556 | Classification_binary: 2.5451 | Regression_cos_phi: 0.6163 | Regression_energy: 1.3819 | Regression_eta: 0.6950 | Regression_pt: 1.4595 | Regression_sin_phi: 0.5074 | LR: 4.80e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6556s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1605 | max_loss: 0.0037 | max_backward: 0.2025 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:36:54,503] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:37:35,038] INFO : Step 27600/50000 rank0 | Train Loss: 7.2782 | Train Loss Unweighted: 2.7160 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4367 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1214 | Regression_eta: 0.0076 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8747 | Classification_binary: 2.4367 | Regression_cos_phi: 0.4478 | Regression_energy: 1.3298 | Regression_eta: 0.3758 | Regression_pt: 1.4266 | Regression_sin_phi: 0.3869 | LR: 4.77e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6995s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1405 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.3016 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3998.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:37:35,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:38:15,694] INFO : Step 27700/50000 rank0 | Train Loss: 7.4335 | Train Loss Unweighted: 2.6426 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3520 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1258 | Regression_eta: 0.0099 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8414 | Classification_binary: 2.3520 | Regression_cos_phi: 0.4946 | Regression_energy: 1.3786 | Regression_eta: 0.4844 | Regression_pt: 1.4792 | Regression_sin_phi: 0.4033 | LR: 4.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6010s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0927 | max_loss: 0.0034 | max_backward: 0.2023 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3995.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:38:15,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:38:56,886] INFO : Step 27800/50000 rank0 | Train Loss: 7.8478 | Train Loss Unweighted: 2.8224 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.5055 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1384 | Regression_eta: 0.0102 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8466 | Classification_binary: 2.5055 | Regression_cos_phi: 0.4231 | Regression_energy: 1.5162 | Regression_eta: 0.5003 | Regression_pt: 1.6503 | Regression_sin_phi: 0.4057 | LR: 4.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5659s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0031 | max_backward: 0.2143 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.3715 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4944.4 | uss_MB: 4185.4 | child_rss_MB: 3997.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:38:57,098] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:39:38,169] INFO : Step 27900/50000 rank0 | Train Loss: 7.1909 | Train Loss Unweighted: 2.6822 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4104 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1211 | Regression_eta: 0.0084 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8345 | Classification_binary: 2.4104 | Regression_cos_phi: 0.4672 | Regression_energy: 1.3270 | Regression_eta: 0.4142 | Regression_pt: 1.3347 | Regression_sin_phi: 0.4029 | LR: 4.68e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8285s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0252 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.1390 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4942.9 | uss_MB: 4183.9 | child_rss_MB: 3995.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:39:38,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:40:19,331] INFO : Step 28000/50000 rank0 | Train Loss: 7.8859 | Train Loss Unweighted: 2.7710 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4478 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1435 | Regression_eta: 0.0100 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8516 | Classification_binary: 2.4478 | Regression_cos_phi: 0.4599 | Regression_energy: 1.5718 | Regression_eta: 0.4939 | Regression_pt: 1.6616 | Regression_sin_phi: 0.3992 | LR: 4.66e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.6439s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0491 | max_forward: 0.0775 | max_loss: 0.0029 | max_backward: 0.3096 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4942.9 | uss_MB: 4185.1 | child_rss_MB: 4015.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:40:19,544] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:40:19,564] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 14:40:19,737] INFO : Running validation on rank0 +rank0 [2026-10-28 14:40:19,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:40:19,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:40:21,082] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 14:40:21,227] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:40:21,358] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:40:21,433] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:40:21,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:40:21,571] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:40:21,641] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:40:21,710] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:40:21,822] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:40:21,955] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:40:22,086] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:40:22,158] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:40:22,227] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:40:22,295] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:40:22,364] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:40:22,434] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:40:22,530] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:40:22,537] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:40:22,540] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:40:22,543] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:40:22,543] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:40:22,546] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:40:22,549] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:40:22,550] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:40:22,553] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:40:22,554] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:40:22,557] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:40:22,559] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:40:22,561] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:40:22,563] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:40:22,566] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:40:22,567] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:40:30,131] INFO : VALIDATION | Step=28000/50000 | Train Loss=7.8859 | Valid Loss=7.8292 | Train Loss Unweighted=2.7710 | Valid Loss Unweighted=2.7918 | Stale=0 | ETA=163.4m +rank0 [2026-10-28 14:40:30,141] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:40:30,200] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:40:30,204] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:40:30,345] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:40:32,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:40:33,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:40:33,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:40:34,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:40:34,504] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:40:34,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:40:34,588] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:40:34,591] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:40:34,667] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:40:36,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:40:37,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:40:37,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:40:38,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:40:38,695] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:40:38,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:40:38,777] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:40:38,779] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:40:38,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:40:40,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:40:41,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:40:42,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:40:42,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:40:42,881] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:40:42,889] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:40:43,150] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:40:43,553] INFO : Plotted number of elements +rank0 [2026-10-28 14:40:44,996] INFO : Plotted jets +rank0 [2026-10-28 14:40:46,084] INFO : Plotted jet ratio +rank0 [2026-10-28 14:40:46,116] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:40:46,349] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:40:47,312] INFO : Plotted number of elements +rank0 [2026-10-28 14:40:48,706] INFO : Plotted jets +rank0 [2026-10-28 14:40:49,815] INFO : Plotted jet ratio +rank0 [2026-10-28 14:40:49,848] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 14:40:50,057] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_28000/cld_edm_qq_pf +rank0 [2026-10-28 14:40:50,419] INFO : Plotted number of elements +rank0 [2026-10-28 14:40:51,802] INFO : Plotted jets +rank0 [2026-10-28 14:40:52,872] INFO : Plotted jet ratio +rank0 [2026-10-28 14:41:34,296] INFO : Step 28100/50000 rank0 | Train Loss: 7.7474 | Train Loss Unweighted: 2.7896 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4996 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1240 | Regression_eta: 0.0141 | Regression_pt: 0.1197 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8587 | Classification_binary: 2.4996 | Regression_cos_phi: 0.4168 | Regression_energy: 1.3589 | Regression_eta: 0.6908 | Regression_pt: 1.4357 | Regression_sin_phi: 0.4868 | LR: 4.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6986s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0790 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3098 | max_optimizer_step: 0.1372 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5147.9 | uss_MB: 4393.1 | child_rss_MB: 4015.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:41:34,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:42:16,002] INFO : Step 28200/50000 rank0 | Train Loss: 7.9917 | Train Loss Unweighted: 2.8371 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.5477 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1207 | Regression_eta: 0.0147 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9598 | Classification_binary: 2.5477 | Regression_cos_phi: 0.5665 | Regression_energy: 1.3225 | Regression_eta: 0.7203 | Regression_pt: 1.4150 | Regression_sin_phi: 0.4600 | LR: 4.60e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6791s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1044 | max_loss: 0.0037 | max_backward: 0.2934 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.3423 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5147.9 | uss_MB: 4395.3 | child_rss_MB: 3992.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:42:16,212] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:42:57,314] INFO : Step 28300/50000 rank0 | Train Loss: 8.2479 | Train Loss Unweighted: 2.8117 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4892 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1406 | Regression_eta: 0.0159 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8546 | Classification_binary: 2.4892 | Regression_cos_phi: 0.4985 | Regression_energy: 1.5403 | Regression_eta: 0.7802 | Regression_pt: 1.5910 | Regression_sin_phi: 0.4940 | LR: 4.57e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6661s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0773 | max_loss: 0.0036 | max_backward: 0.2436 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.4067 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5014.9 | uss_MB: 4259.3 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:42:57,527] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:43:39,333] INFO : Step 28400/50000 rank0 | Train Loss: 7.6846 | Train Loss Unweighted: 2.7839 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4764 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1347 | Regression_eta: 0.0097 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8111 | Classification_binary: 2.4764 | Regression_cos_phi: 0.4418 | Regression_energy: 1.4757 | Regression_eta: 0.4786 | Regression_pt: 1.5931 | Regression_sin_phi: 0.4078 | LR: 4.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6473s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.1029 | max_backward: 0.2031 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3175 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5015.9 | uss_MB: 4260.6 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:43:39,545] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:44:20,010] INFO : Step 28500/50000 rank0 | Train Loss: 8.2545 | Train Loss Unweighted: 2.8759 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.5617 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1335 | Regression_eta: 0.0177 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7759 | Classification_binary: 2.5617 | Regression_cos_phi: 0.5886 | Regression_energy: 1.4630 | Regression_eta: 0.8702 | Regression_pt: 1.5739 | Regression_sin_phi: 0.4213 | LR: 4.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5660s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0294 | max_backward: 0.2032 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3193 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5015.9 | uss_MB: 4260.8 | child_rss_MB: 3992.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:44:20,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:01,998] INFO : Step 28600/50000 rank0 | Train Loss: 7.7300 | Train Loss Unweighted: 2.8020 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.4999 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1327 | Regression_eta: 0.0111 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7887 | Classification_binary: 2.4999 | Regression_cos_phi: 0.4323 | Regression_energy: 1.4542 | Regression_eta: 0.5434 | Regression_pt: 1.5299 | Regression_sin_phi: 0.4815 | LR: 4.49e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6821s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2732 | max_optimizer_step: 0.1357 | max_tensorboard_logging: 0.3835 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5015.9 | uss_MB: 4260.9 | child_rss_MB: 3993.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:45:02,211] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:42,712] INFO : Step 28700/50000 rank0 | Train Loss: 7.6298 | Train Loss Unweighted: 2.7918 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5067 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1229 | Regression_eta: 0.0141 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8228 | Classification_binary: 2.5067 | Regression_cos_phi: 0.4431 | Regression_energy: 1.3462 | Regression_eta: 0.6931 | Regression_pt: 1.4116 | Regression_sin_phi: 0.4062 | LR: 4.46e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5528s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0119 | max_backward: 0.2030 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3174 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5015.9 | uss_MB: 4260.9 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:45:42,926] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:46:24,351] INFO : Step 28800/50000 rank0 | Train Loss: 8.1128 | Train Loss Unweighted: 2.8401 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5096 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1456 | Regression_eta: 0.0119 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8217 | Classification_binary: 2.5096 | Regression_cos_phi: 0.4667 | Regression_energy: 1.5950 | Regression_eta: 0.5830 | Regression_pt: 1.7010 | Regression_sin_phi: 0.4359 | LR: 4.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5927s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1860 | max_loss: 0.0035 | max_backward: 0.2025 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.2870 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5015.9 | uss_MB: 4260.9 | child_rss_MB: 3992.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:46:24,568] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:04,895] INFO : Step 28900/50000 rank0 | Train Loss: 8.0833 | Train Loss Unweighted: 2.9385 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.6158 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1433 | Regression_eta: 0.0109 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7860 | Classification_binary: 2.6158 | Regression_cos_phi: 0.4850 | Regression_energy: 1.5702 | Regression_eta: 0.5339 | Regression_pt: 1.6506 | Regression_sin_phi: 0.4417 | LR: 4.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6157s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.2217 | max_optimizer_step: 0.1333 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5015.6 | uss_MB: 4259.4 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:47:05,109] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:45,887] INFO : Step 29000/50000 rank0 | Train Loss: 7.6657 | Train Loss Unweighted: 2.7995 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4936 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1345 | Regression_eta: 0.0081 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8545 | Classification_binary: 2.4936 | Regression_cos_phi: 0.4704 | Regression_energy: 1.4741 | Regression_eta: 0.3978 | Regression_pt: 1.5831 | Regression_sin_phi: 0.3921 | LR: 4.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6983s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1040 | max_loss: 0.0034 | max_backward: 0.3892 | max_optimizer_step: 0.1319 | max_tensorboard_logging: 0.2842 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5016.6 | uss_MB: 4262.8 | child_rss_MB: 4018.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:47:46,100] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:46,120] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 14:47:46,294] INFO : Running validation on rank0 +rank0 [2026-10-28 14:47:46,294] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:47:46,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:47:47,554] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 14:47:47,701] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:47:47,834] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:47:47,909] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:47:47,979] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:47:48,047] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:47:48,118] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:47:48,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:47:48,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:47:48,432] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:47:48,566] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:47:48,637] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:47:48,706] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:47:48,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:47:48,847] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:47:48,916] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:47:49,021] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:47:49,028] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:47:49,032] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:47:49,035] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:47:49,037] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:47:49,040] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:47:49,041] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:47:49,044] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:47:49,047] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:47:49,048] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:47:49,051] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:47:49,054] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:47:49,055] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:47:49,058] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:47:49,060] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:47:49,062] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:47:56,650] INFO : VALIDATION | Step=29000/50000 | Train Loss=7.6657 | Valid Loss=7.7437 | Train Loss Unweighted=2.7995 | Valid Loss Unweighted=2.7686 | Stale=0 | ETA=156.0m +rank0 [2026-10-28 14:47:56,659] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:47:56,739] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:47:56,742] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:47:56,938] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:47:59,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:47:59,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:48:00,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:48:01,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:48:01,341] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:48:01,344] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:48:01,400] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:48:01,403] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:48:01,516] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:48:03,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:48:03,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:48:04,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:48:05,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:48:05,378] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:48:05,381] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:48:05,435] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:48:05,438] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:48:05,542] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:48:07,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:48:08,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:48:08,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:48:09,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:48:09,574] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:48:09,598] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:48:09,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:48:10,261] INFO : Plotted number of elements +rank0 [2026-10-28 14:48:11,699] INFO : Plotted jets +rank0 [2026-10-28 14:48:12,792] INFO : Plotted jet ratio +rank0 [2026-10-28 14:48:12,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:48:13,072] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:48:14,011] INFO : Plotted number of elements +rank0 [2026-10-28 14:48:15,401] INFO : Plotted jets +rank0 [2026-10-28 14:48:16,512] INFO : Plotted jet ratio +rank0 [2026-10-28 14:48:16,546] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 14:48:16,751] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_29000/cld_edm_qq_pf +rank0 [2026-10-28 14:48:17,112] INFO : Plotted number of elements +rank0 [2026-10-28 14:48:18,486] INFO : Plotted jets +rank0 [2026-10-28 14:48:19,551] INFO : Plotted jet ratio +rank0 [2026-10-28 14:49:00,522] INFO : Step 29100/50000 rank0 | Train Loss: 7.2658 | Train Loss Unweighted: 2.6978 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4216 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1199 | Regression_eta: 0.0095 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7767 | Classification_binary: 2.4216 | Regression_cos_phi: 0.4540 | Regression_energy: 1.3133 | Regression_eta: 0.4662 | Regression_pt: 1.4012 | Regression_sin_phi: 0.4329 | LR: 4.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7259s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0031 | max_backward: 0.3076 | max_optimizer_step: 0.1370 | max_tensorboard_logging: 0.3836 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5209.6 | uss_MB: 4454.7 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:49:00,734] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:49:42,613] INFO : Step 29200/50000 rank0 | Train Loss: 7.7446 | Train Loss Unweighted: 2.7666 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4489 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1413 | Regression_eta: 0.0095 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8346 | Classification_binary: 2.4489 | Regression_cos_phi: 0.4445 | Regression_energy: 1.5483 | Regression_eta: 0.4659 | Regression_pt: 1.6408 | Regression_sin_phi: 0.3616 | LR: 4.33e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6704s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2497 | max_optimizer_step: 0.1362 | max_tensorboard_logging: 0.3845 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5209.6 | uss_MB: 4454.7 | child_rss_MB: 3998.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:49:42,827] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:50:23,505] INFO : Step 29300/50000 rank0 | Train Loss: 7.7195 | Train Loss Unweighted: 2.8341 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5276 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1338 | Regression_eta: 0.0098 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8547 | Classification_binary: 2.5276 | Regression_cos_phi: 0.4052 | Regression_energy: 1.4659 | Regression_eta: 0.4829 | Regression_pt: 1.5882 | Regression_sin_phi: 0.3949 | LR: 4.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6471s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1778 | max_loss: 0.0031 | max_backward: 0.2021 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3126 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5087.9 | uss_MB: 4332.7 | child_rss_MB: 3995.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:50:23,729] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:04,597] INFO : Step 29400/50000 rank0 | Train Loss: 7.6670 | Train Loss Unweighted: 2.7807 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4831 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1284 | Regression_eta: 0.0074 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8691 | Classification_binary: 2.4831 | Regression_cos_phi: 0.5660 | Regression_energy: 1.4072 | Regression_eta: 0.3653 | Regression_pt: 1.5348 | Regression_sin_phi: 0.4416 | LR: 4.27e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6726s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.2764 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5088.9 | uss_MB: 4333.8 | child_rss_MB: 3999.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:51:04,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:45,925] INFO : Step 29500/50000 rank0 | Train Loss: 7.5591 | Train Loss Unweighted: 2.6428 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3435 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1301 | Regression_eta: 0.0085 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8622 | Classification_binary: 2.3435 | Regression_cos_phi: 0.4892 | Regression_energy: 1.4250 | Regression_eta: 0.4197 | Regression_pt: 1.5285 | Regression_sin_phi: 0.4911 | LR: 4.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7111s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3081 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3793 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5088.9 | uss_MB: 4333.9 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:51:46,138] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:52:26,978] INFO : Step 29600/50000 rank0 | Train Loss: 7.5619 | Train Loss Unweighted: 2.7819 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4856 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1296 | Regression_eta: 0.0091 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8423 | Classification_binary: 2.4856 | Regression_cos_phi: 0.4602 | Regression_energy: 1.4199 | Regression_eta: 0.4463 | Regression_pt: 1.5197 | Regression_sin_phi: 0.3879 | LR: 4.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7920s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1840 | max_loss: 0.0031 | max_backward: 0.2021 | max_optimizer_step: 0.2401 | max_tensorboard_logging: 0.3326 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 5088.9 | uss_MB: 4334.0 | child_rss_MB: 4004.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:52:27,191] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:08,715] INFO : Step 29700/50000 rank0 | Train Loss: 8.3498 | Train Loss Unweighted: 2.9355 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5898 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1539 | Regression_eta: 0.0109 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8089 | Classification_binary: 2.5898 | Regression_cos_phi: 0.5355 | Regression_energy: 1.6863 | Regression_eta: 0.5356 | Regression_pt: 1.7930 | Regression_sin_phi: 0.4007 | LR: 4.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7444s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1128 | max_loss: 0.0033 | max_backward: 0.3950 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.2875 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5088.9 | uss_MB: 4336.4 | child_rss_MB: 4026.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:53:08,927] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:49,620] INFO : Step 29800/50000 rank0 | Train Loss: 7.0682 | Train Loss Unweighted: 2.7709 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4958 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1211 | Regression_eta: 0.0069 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8180 | Classification_binary: 2.4958 | Regression_cos_phi: 0.3288 | Regression_energy: 1.3273 | Regression_eta: 0.3398 | Regression_pt: 1.4336 | Regression_sin_phi: 0.3248 | LR: 4.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6100s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2294 | max_optimizer_step: 0.1314 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5088.9 | uss_MB: 4334.4 | child_rss_MB: 3998.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:53:49,832] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:54:30,991] INFO : Step 29900/50000 rank0 | Train Loss: 8.6388 | Train Loss Unweighted: 2.8845 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.5485 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1459 | Regression_eta: 0.0190 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8066 | Classification_binary: 2.5485 | Regression_cos_phi: 0.5477 | Regression_energy: 1.5987 | Regression_eta: 0.9336 | Regression_pt: 1.6439 | Regression_sin_phi: 0.5599 | LR: 4.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7372s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1038 | max_loss: 0.0032 | max_backward: 0.3595 | max_optimizer_step: 0.1318 | max_tensorboard_logging: 0.3413 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5087.7 | uss_MB: 4334.8 | child_rss_MB: 3997.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:54:31,206] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:11,921] INFO : Step 30000/50000 rank0 | Train Loss: 7.9552 | Train Loss Unweighted: 2.7797 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4734 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1334 | Regression_eta: 0.0127 | Regression_pt: 0.1266 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8158 | Classification_binary: 2.4734 | Regression_cos_phi: 0.4973 | Regression_energy: 1.4614 | Regression_eta: 0.6238 | Regression_pt: 1.5187 | Regression_sin_phi: 0.5648 | LR: 4.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5875s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0954 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.2021 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.7 | uss_MB: 4333.9 | child_rss_MB: 3998.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:55:12,143] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:12,161] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 14:55:12,336] INFO : Running validation on rank0 +rank0 [2026-10-28 14:55:12,336] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:55:12,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:55:13,652] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 14:55:13,798] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:55:13,929] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:55:14,003] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:55:14,073] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:55:14,142] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:55:14,212] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:55:14,281] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:55:14,394] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:55:14,525] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:55:14,656] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:55:14,727] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:55:14,797] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:55:14,865] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:55:14,935] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:55:15,004] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:55:15,121] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:55:15,128] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:55:15,131] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:55:15,134] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:55:15,136] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:55:15,138] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:55:15,141] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:55:15,143] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:55:15,144] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:55:15,148] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:55:15,150] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:55:15,153] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:55:15,157] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:55:15,159] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:55:15,160] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:55:15,163] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:55:22,696] INFO : VALIDATION | Step=30000/50000 | Train Loss=7.9552 | Valid Loss=7.7615 | Train Loss Unweighted=2.7797 | Valid Loss Unweighted=2.7671 | Stale=1 | ETA=148.5m +rank0 [2026-10-28 14:55:22,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:55:22,772] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:55:22,776] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 14:55:22,946] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:55:24,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:55:25,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:55:26,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:55:26,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:55:26,801] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:55:26,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:55:26,875] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:55:26,878] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:55:26,968] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:55:28,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:55:29,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:55:30,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:55:30,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:55:30,851] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:55:30,855] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:55:30,910] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 14:55:30,912] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 14:55:31,018] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:55:32,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:55:33,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:55:34,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:55:34,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:55:35,015] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:55:35,022] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:55:35,281] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-10-28 14:55:35,688] INFO : Plotted number of elements +rank0 [2026-10-28 14:55:37,127] INFO : Plotted jets +rank0 [2026-10-28 14:55:38,774] INFO : Plotted jet ratio +rank0 [2026-10-28 14:55:38,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:55:39,038] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 14:55:39,429] INFO : Plotted number of elements +rank0 [2026-10-28 14:55:40,831] INFO : Plotted jets +rank0 [2026-10-28 14:55:41,951] INFO : Plotted jet ratio +rank0 [2026-10-28 14:55:41,985] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 14:55:42,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_30000/cld_edm_qq_pf +rank0 [2026-10-28 14:55:42,559] INFO : Plotted number of elements +rank0 [2026-10-28 14:55:43,943] INFO : Plotted jets +rank0 [2026-10-28 14:55:45,005] INFO : Plotted jet ratio +rank0 [2026-10-28 14:56:27,215] INFO : Step 30100/50000 rank0 | Train Loss: 7.8092 | Train Loss Unweighted: 2.7948 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4808 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1379 | Regression_eta: 0.0099 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8896 | Classification_binary: 2.4808 | Regression_cos_phi: 0.5110 | Regression_energy: 1.5115 | Regression_eta: 0.4864 | Regression_pt: 1.6163 | Regression_sin_phi: 0.3136 | LR: 4.08e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5735s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0154 | max_backward: 0.2027 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3146 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5279.3 | uss_MB: 4523.1 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:56:27,426] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:57:07,554] INFO : Step 30200/50000 rank0 | Train Loss: 7.8516 | Train Loss Unweighted: 2.7367 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4411 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1283 | Regression_eta: 0.0186 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8072 | Classification_binary: 2.4411 | Regression_cos_phi: 0.4632 | Regression_energy: 1.4053 | Regression_eta: 0.9126 | Regression_pt: 1.4210 | Regression_sin_phi: 0.4011 | LR: 4.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6577s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0151 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3091 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5279.3 | uss_MB: 4523.1 | child_rss_MB: 4024.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:57:07,767] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:57:49,220] INFO : Step 30300/50000 rank0 | Train Loss: 8.0430 | Train Loss Unweighted: 2.8609 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.5496 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1352 | Regression_eta: 0.0162 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8482 | Classification_binary: 2.5496 | Regression_cos_phi: 0.4510 | Regression_energy: 1.4809 | Regression_eta: 0.7975 | Regression_pt: 1.5530 | Regression_sin_phi: 0.3628 | LR: 4.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6752s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.2201 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3875 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5279.3 | uss_MB: 4523.1 | child_rss_MB: 3998.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:57:49,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:58:31,509] INFO : Step 30400/50000 rank0 | Train Loss: 8.2332 | Train Loss Unweighted: 2.7203 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4006 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1375 | Regression_eta: 0.0210 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8657 | Classification_binary: 2.4006 | Regression_cos_phi: 0.4740 | Regression_energy: 1.5070 | Regression_eta: 1.0340 | Regression_pt: 1.5518 | Regression_sin_phi: 0.4001 | LR: 4.00e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1497 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5159.8 | uss_MB: 4403.0 | child_rss_MB: 3998.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:58:31,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:12,545] INFO : Step 30500/50000 rank0 | Train Loss: 8.1211 | Train Loss Unweighted: 2.7258 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4250 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1255 | Regression_eta: 0.0227 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8319 | Classification_binary: 2.4250 | Regression_cos_phi: 0.4696 | Regression_energy: 1.3751 | Regression_eta: 1.1138 | Regression_pt: 1.4493 | Regression_sin_phi: 0.4565 | LR: 3.98e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7341s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5159.8 | uss_MB: 4403.0 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:59:12,758] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:54,647] INFO : Step 30600/50000 rank0 | Train Loss: 7.5510 | Train Loss Unweighted: 2.7837 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4941 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1253 | Regression_eta: 0.0114 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8610 | Classification_binary: 2.4941 | Regression_cos_phi: 0.4442 | Regression_energy: 1.3724 | Regression_eta: 0.5598 | Regression_pt: 1.4700 | Regression_sin_phi: 0.3494 | LR: 3.95e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0446 | max_backward: 0.2031 | max_optimizer_step: 0.1368 | max_tensorboard_logging: 0.3083 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5159.8 | uss_MB: 4403.2 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:59:54,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:00:35,697] INFO : Step 30700/50000 rank0 | Train Loss: 7.9790 | Train Loss Unweighted: 2.8122 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4980 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1392 | Regression_eta: 0.0142 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8294 | Classification_binary: 2.4980 | Regression_cos_phi: 0.4758 | Regression_energy: 1.5256 | Regression_eta: 0.6997 | Regression_pt: 1.5571 | Regression_sin_phi: 0.3935 | LR: 3.92e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6313s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0097 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3071 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5159.8 | uss_MB: 4403.2 | child_rss_MB: 4021.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:00:35,910] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:17,853] INFO : Step 30800/50000 rank0 | Train Loss: 7.3387 | Train Loss Unweighted: 2.7645 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4872 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1212 | Regression_eta: 0.0107 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8057 | Classification_binary: 2.4872 | Regression_cos_phi: 0.4280 | Regression_energy: 1.3285 | Regression_eta: 0.5240 | Regression_pt: 1.3916 | Regression_sin_phi: 0.3737 | LR: 3.90e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7085s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1843 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.2848 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5159.8 | uss_MB: 4403.4 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:01:18,082] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:59,346] INFO : Step 30900/50000 rank0 | Train Loss: 7.3086 | Train Loss Unweighted: 2.7052 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4285 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1193 | Regression_eta: 0.0099 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8570 | Classification_binary: 2.4285 | Regression_cos_phi: 0.3481 | Regression_energy: 1.3069 | Regression_eta: 0.4881 | Regression_pt: 1.3960 | Regression_sin_phi: 0.4840 | LR: 3.87e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7842s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1456 | max_loss: 0.0035 | max_backward: 0.3893 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.2928 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5160.8 | uss_MB: 4405.8 | child_rss_MB: 4026.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 15:01:59,558] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:02:39,670] INFO : Step 31000/50000 rank0 | Train Loss: 7.8594 | Train Loss Unweighted: 2.7765 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4650 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1337 | Regression_eta: 0.0103 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8798 | Classification_binary: 2.4650 | Regression_cos_phi: 0.4848 | Regression_energy: 1.4655 | Regression_eta: 0.5074 | Regression_pt: 1.6142 | Regression_sin_phi: 0.4427 | LR: 3.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5333s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0055 | max_backward: 0.2031 | max_optimizer_step: 0.1301 | max_tensorboard_logging: 0.3298 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5158.9 | uss_MB: 4402.9 | child_rss_MB: 3998.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:02:39,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:02:39,924] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 15:02:40,107] INFO : Running validation on rank0 +rank0 [2026-10-28 15:02:40,107] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:02:40,107] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:02:41,543] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 15:02:41,687] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:02:41,819] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:02:41,894] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:02:41,963] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:02:42,032] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:02:42,102] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:02:42,171] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:02:42,285] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:02:42,416] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:02:42,548] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:02:42,620] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:02:42,689] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:02:42,757] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:02:42,826] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:02:42,895] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:02:43,011] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:02:43,018] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:02:43,020] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:02:43,022] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:02:43,024] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:02:43,025] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:02:43,028] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:02:43,031] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:02:43,032] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:02:43,035] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:02:43,035] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:02:43,038] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:02:43,041] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:02:43,042] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:02:43,044] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:02:43,045] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:02:50,527] INFO : VALIDATION | Step=31000/50000 | Train Loss=7.8594 | Valid Loss=7.6697 | Train Loss Unweighted=2.7765 | Valid Loss Unweighted=2.7551 | Stale=0 | ETA=141.1m +rank0 [2026-10-28 15:02:50,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:02:50,637] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:02:50,641] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:02:50,810] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:02:52,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:02:53,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:02:54,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:02:54,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:02:54,770] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:02:54,773] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:02:54,836] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:02:54,839] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:02:54,939] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:02:56,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:02:57,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:02:58,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:02:58,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:02:58,878] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:02:58,881] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:02:58,945] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:02:58,948] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:02:59,052] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:03:00,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:03:01,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:03:02,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:03:02,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:03:03,011] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:03:03,019] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:03:03,278] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:03:03,682] INFO : Plotted number of elements +rank0 [2026-10-28 15:03:05,120] INFO : Plotted jets +rank0 [2026-10-28 15:03:06,205] INFO : Plotted jet ratio +rank0 [2026-10-28 15:03:06,238] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:03:06,467] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:03:06,857] INFO : Plotted number of elements +rank0 [2026-10-28 15:03:08,827] INFO : Plotted jets +rank0 [2026-10-28 15:03:09,932] INFO : Plotted jet ratio +rank0 [2026-10-28 15:03:09,965] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 15:03:10,178] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_31000/cld_edm_qq_pf +rank0 [2026-10-28 15:03:10,540] INFO : Plotted number of elements +rank0 [2026-10-28 15:03:11,924] INFO : Plotted jets +rank0 [2026-10-28 15:03:12,988] INFO : Plotted jet ratio +rank0 [2026-10-28 15:03:55,360] INFO : Step 31100/50000 rank0 | Train Loss: 7.7979 | Train Loss Unweighted: 2.6861 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3795 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1342 | Regression_eta: 0.0154 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7933 | Classification_binary: 2.3795 | Regression_cos_phi: 0.5001 | Regression_energy: 1.4708 | Regression_eta: 0.7581 | Regression_pt: 1.5216 | Regression_sin_phi: 0.3745 | LR: 3.82e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6569s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0193 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3099 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.2790 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5355.2 | uss_MB: 4600.6 | child_rss_MB: 4030.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:03:55,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:04:37,224] INFO : Step 31200/50000 rank0 | Train Loss: 7.5580 | Train Loss Unweighted: 2.7084 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4108 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1312 | Regression_eta: 0.0103 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8069 | Classification_binary: 2.4108 | Regression_cos_phi: 0.4716 | Regression_energy: 1.4380 | Regression_eta: 0.5063 | Regression_pt: 1.5027 | Regression_sin_phi: 0.4218 | LR: 3.79e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7250s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0037 | max_backward: 0.3091 | max_optimizer_step: 0.1362 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5355.2 | uss_MB: 4600.6 | child_rss_MB: 4000.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:04:37,437] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 3% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:05:18,840] INFO : Step 31300/50000 rank0 | Train Loss: 7.7047 | Train Loss Unweighted: 2.6989 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3771 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1425 | Regression_eta: 0.0075 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8819 | Classification_binary: 2.3771 | Regression_cos_phi: 0.4567 | Regression_energy: 1.5615 | Regression_eta: 0.3682 | Regression_pt: 1.6834 | Regression_sin_phi: 0.3760 | LR: 3.76e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7907s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1050 | max_loss: 0.0035 | max_backward: 0.3724 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.3308 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5355.2 | uss_MB: 4602.7 | child_rss_MB: 3997.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 15:05:19,054] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:05:59,462] INFO : Step 31400/50000 rank0 | Train Loss: 7.8413 | Train Loss Unweighted: 2.7043 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3956 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1344 | Regression_eta: 0.0125 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7880 | Classification_binary: 2.3956 | Regression_cos_phi: 0.5036 | Regression_energy: 1.4730 | Regression_eta: 0.6154 | Regression_pt: 1.5530 | Regression_sin_phi: 0.5126 | LR: 3.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7431s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3102 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.4013 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 5235.6 | uss_MB: 4479.3 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:05:59,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:06:41,109] INFO : Step 31500/50000 rank0 | Train Loss: 8.0302 | Train Loss Unweighted: 2.8475 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5376 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1323 | Regression_eta: 0.0134 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8800 | Classification_binary: 2.5376 | Regression_cos_phi: 0.4667 | Regression_energy: 1.4493 | Regression_eta: 0.6566 | Regression_pt: 1.5743 | Regression_sin_phi: 0.4657 | LR: 3.71e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1788 | max_loss: 0.0035 | max_backward: 0.2028 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5235.6 | uss_MB: 4479.7 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:06:41,338] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:07:21,939] INFO : Step 31600/50000 rank0 | Train Loss: 7.8913 | Train Loss Unweighted: 2.8011 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4812 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1414 | Regression_eta: 0.0094 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 2.4812 | Regression_cos_phi: 0.5051 | Regression_energy: 1.5494 | Regression_eta: 0.4614 | Regression_pt: 1.6490 | Regression_sin_phi: 0.4317 | LR: 3.69e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6202s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0082 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3079 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2816 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5235.6 | uss_MB: 4479.7 | child_rss_MB: 4018.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:07:22,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:03,843] INFO : Step 31700/50000 rank0 | Train Loss: 8.3653 | Train Loss Unweighted: 2.9146 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.5871 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1414 | Regression_eta: 0.0139 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8978 | Classification_binary: 2.5871 | Regression_cos_phi: 0.5229 | Regression_energy: 1.5490 | Regression_eta: 0.6821 | Regression_pt: 1.6544 | Regression_sin_phi: 0.4720 | LR: 3.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.2350s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1004 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.3930 | max_optimizer_step: 0.1358 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5236.6 | uss_MB: 4482.3 | child_rss_MB: 3995.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:08:04,056] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:44,932] INFO : Step 31800/50000 rank0 | Train Loss: 7.4813 | Train Loss Unweighted: 2.7657 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4805 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1218 | Regression_eta: 0.0095 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7805 | Classification_binary: 2.4805 | Regression_cos_phi: 0.5004 | Regression_energy: 1.3343 | Regression_eta: 0.4659 | Regression_pt: 1.4748 | Regression_sin_phi: 0.4448 | LR: 3.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6539s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0515 | max_forward: 0.0776 | max_loss: 0.0030 | max_backward: 0.3086 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.2859 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5237.6 | uss_MB: 4480.3 | child_rss_MB: 4016.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:08:45,146] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:09:25,950] INFO : Step 31900/50000 rank0 | Train Loss: 7.2305 | Train Loss Unweighted: 2.7003 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.4220 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1214 | Regression_eta: 0.0112 | Regression_pt: 0.1181 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7497 | Classification_binary: 2.4220 | Regression_cos_phi: 0.3491 | Regression_energy: 1.3300 | Regression_eta: 0.5501 | Regression_pt: 1.4162 | Regression_sin_phi: 0.4135 | LR: 3.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6956s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2791 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5237.6 | uss_MB: 4480.3 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:09:26,164] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:07,604] INFO : Step 32000/50000 rank0 | Train Loss: 7.6841 | Train Loss Unweighted: 2.7223 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4355 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1215 | Regression_eta: 0.0126 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7985 | Classification_binary: 2.4355 | Regression_cos_phi: 0.6335 | Regression_energy: 1.3311 | Regression_eta: 0.6219 | Regression_pt: 1.4361 | Regression_sin_phi: 0.4275 | LR: 3.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7240s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1842 | max_loss: 0.0035 | max_backward: 0.2026 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.2971 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5238.6 | uss_MB: 4479.2 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:10:07,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:07,839] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 15:10:08,018] INFO : Running validation on rank0 +rank0 [2026-10-28 15:10:08,019] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:10:08,019] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:10:09,356] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 15:10:09,503] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:10:09,637] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:10:09,711] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:10:09,780] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:10:09,848] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:10:09,918] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:10:09,987] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:10:10,100] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:10:10,233] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:10:10,365] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:10:10,437] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:10:10,507] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:10:10,575] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:10:10,645] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:10:10,715] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:10:10,811] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:10:10,818] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:10:10,821] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:10:10,824] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:10:10,825] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:10:10,828] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:10:10,831] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:10:10,832] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:10:10,835] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:10:10,838] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:10:10,841] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:10:10,844] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:10:10,845] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:10:10,847] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:10:10,848] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:10:10,852] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:10:18,413] INFO : VALIDATION | Step=32000/50000 | Train Loss=7.6841 | Valid Loss=7.6540 | Train Loss Unweighted=2.7223 | Valid Loss Unweighted=2.7312 | Stale=0 | ETA=133.7m +rank0 [2026-10-28 15:10:18,423] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:10:18,608] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:10:18,612] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:10:18,795] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:10:20,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:10:21,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:10:22,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:10:22,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:10:23,134] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:10:23,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:10:23,215] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:10:23,218] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:10:23,307] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:10:25,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:10:25,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:10:26,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:10:27,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:10:27,150] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:10:27,153] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:10:27,220] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:10:27,223] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:10:27,316] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:10:29,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:10:29,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:10:30,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:10:31,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:10:31,326] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:10:31,333] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:10:31,595] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:10:32,001] INFO : Plotted number of elements +rank0 [2026-10-28 15:10:33,435] INFO : Plotted jets +rank0 [2026-10-28 15:10:34,529] INFO : Plotted jet ratio +rank0 [2026-10-28 15:10:34,563] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:10:34,785] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:10:35,175] INFO : Plotted number of elements +rank0 [2026-10-28 15:10:37,152] INFO : Plotted jets +rank0 [2026-10-28 15:10:38,257] INFO : Plotted jet ratio +rank0 [2026-10-28 15:10:38,290] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 15:10:38,494] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_32000/cld_edm_qq_pf +rank0 [2026-10-28 15:10:38,859] INFO : Plotted number of elements +rank0 [2026-10-28 15:10:40,237] INFO : Plotted jets +rank0 [2026-10-28 15:10:41,299] INFO : Plotted jet ratio +rank0 [2026-10-28 15:11:22,413] INFO : Step 32100/50000 rank0 | Train Loss: 7.6895 | Train Loss Unweighted: 2.7110 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3978 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1387 | Regression_eta: 0.0102 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8630 | Classification_binary: 2.3978 | Regression_cos_phi: 0.4430 | Regression_energy: 1.5198 | Regression_eta: 0.5021 | Regression_pt: 1.6033 | Regression_sin_phi: 0.3605 | LR: 3.56e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.7228s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1045 | max_loss: 0.0035 | max_backward: 0.3116 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3319 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5441.1 | uss_MB: 4684.6 | child_rss_MB: 3996.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 15:11:22,627] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:12:03,399] INFO : Step 32200/50000 rank0 | Train Loss: 8.2772 | Train Loss Unweighted: 2.8474 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5065 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1522 | Regression_eta: 0.0106 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8411 | Classification_binary: 2.5065 | Regression_cos_phi: 0.5735 | Regression_energy: 1.6680 | Regression_eta: 0.5191 | Regression_pt: 1.7398 | Regression_sin_phi: 0.4292 | LR: 3.53e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.7169s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.1838 | max_loss: 0.0036 | max_backward: 0.2026 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5441.1 | uss_MB: 4682.5 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:12:03,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:12:45,479] INFO : Step 32300/50000 rank0 | Train Loss: 7.6594 | Train Loss Unweighted: 2.7335 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4416 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1256 | Regression_eta: 0.0091 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9143 | Classification_binary: 2.4416 | Regression_cos_phi: 0.6323 | Regression_energy: 1.3759 | Regression_eta: 0.4497 | Regression_pt: 1.4696 | Regression_sin_phi: 0.3760 | LR: 3.51e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7571s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1757 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.2929 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5441.1 | uss_MB: 4682.6 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:12:45,695] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:13:26,660] INFO : Step 32400/50000 rank0 | Train Loss: 7.4844 | Train Loss Unweighted: 2.7397 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.4431 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1313 | Regression_eta: 0.0094 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8027 | Classification_binary: 2.4431 | Regression_cos_phi: 0.4953 | Regression_energy: 1.4390 | Regression_eta: 0.4631 | Regression_pt: 1.5145 | Regression_sin_phi: 0.3267 | LR: 3.48e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5600s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0062 | max_backward: 0.2031 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.3116 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5309.7 | uss_MB: 4549.2 | child_rss_MB: 3994.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:13:26,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:07,371] INFO : Step 32500/50000 rank0 | Train Loss: 7.7165 | Train Loss Unweighted: 2.7521 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.4544 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1298 | Regression_eta: 0.0121 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7453 | Classification_binary: 2.4544 | Regression_cos_phi: 0.5192 | Regression_energy: 1.4227 | Regression_eta: 0.5935 | Regression_pt: 1.4957 | Regression_sin_phi: 0.4857 | LR: 3.46e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7318s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3102 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3819 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5309.7 | uss_MB: 4549.3 | child_rss_MB: 3997.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:14:07,585] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:48,136] INFO : Step 32600/50000 rank0 | Train Loss: 7.7438 | Train Loss Unweighted: 2.7485 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4537 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1282 | Regression_eta: 0.0106 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8786 | Classification_binary: 2.4537 | Regression_cos_phi: 0.5952 | Regression_energy: 1.4043 | Regression_eta: 0.5222 | Regression_pt: 1.4603 | Regression_sin_phi: 0.4295 | LR: 3.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5582s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0127 | max_backward: 0.2026 | max_optimizer_step: 0.1373 | max_tensorboard_logging: 0.3030 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5309.7 | uss_MB: 4549.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:14:48,350] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:15:29,183] INFO : Step 32700/50000 rank0 | Train Loss: 7.6201 | Train Loss Unweighted: 2.6780 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3871 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1250 | Regression_eta: 0.0127 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8755 | Classification_binary: 2.3871 | Regression_cos_phi: 0.4957 | Regression_energy: 1.3691 | Regression_eta: 0.6224 | Regression_pt: 1.4475 | Regression_sin_phi: 0.4227 | LR: 3.41e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6466s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.3083 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.2894 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5309.7 | uss_MB: 4549.4 | child_rss_MB: 4023.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:15:29,396] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:11,415] INFO : Step 32800/50000 rank0 | Train Loss: 7.4295 | Train Loss Unweighted: 2.7360 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4418 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1285 | Regression_eta: 0.0098 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7995 | Classification_binary: 2.4418 | Regression_cos_phi: 0.4247 | Regression_energy: 1.4085 | Regression_eta: 0.4793 | Regression_pt: 1.5225 | Regression_sin_phi: 0.3531 | LR: 3.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6834s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1642 | max_loss: 0.0036 | max_backward: 0.2026 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5309.7 | uss_MB: 4549.5 | child_rss_MB: 3994.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:16:11,628] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:52,072] INFO : Step 32900/50000 rank0 | Train Loss: 8.4421 | Train Loss Unweighted: 2.8762 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5318 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1501 | Regression_eta: 0.0124 | Regression_pt: 0.1477 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9126 | Classification_binary: 2.5318 | Regression_cos_phi: 0.4950 | Regression_energy: 1.6441 | Regression_eta: 0.6101 | Regression_pt: 1.7717 | Regression_sin_phi: 0.4768 | LR: 3.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6271s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0014 | max_forward: 0.0958 | max_loss: 0.0036 | max_backward: 0.2026 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5310.7 | uss_MB: 4549.5 | child_rss_MB: 3994.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:16:52,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:17:33,811] INFO : Step 33000/50000 rank0 | Train Loss: 7.2547 | Train Loss Unweighted: 2.6361 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3567 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1210 | Regression_eta: 0.0128 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7709 | Classification_binary: 2.3567 | Regression_cos_phi: 0.3896 | Regression_energy: 1.3262 | Regression_eta: 0.6296 | Regression_pt: 1.4091 | Regression_sin_phi: 0.3726 | LR: 3.33e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8937s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0474 | max_forward: 0.1045 | max_loss: 0.0036 | max_backward: 0.3925 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.3935 | max_tensorboard_flush: 0.0133 | Memory: rss_MB: 5311.3 | uss_MB: 4551.4 | child_rss_MB: 3994.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:17:34,024] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:17:34,142] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 15:17:34,315] INFO : Running validation on rank0 +rank0 [2026-10-28 15:17:34,315] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:17:34,315] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:17:35,676] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 15:17:35,823] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:17:35,955] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:17:36,029] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:17:36,098] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:17:36,167] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:17:36,237] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:17:36,306] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:17:36,418] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:17:36,549] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:17:36,681] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:17:36,753] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:17:36,823] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:17:36,891] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:17:36,960] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:17:37,031] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:17:37,129] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:17:37,135] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:17:37,138] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:17:37,141] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:17:37,142] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:17:37,145] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:17:37,148] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:17:37,149] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:17:37,151] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:17:37,155] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:17:37,157] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:17:37,158] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:17:37,161] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:17:37,163] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:17:37,164] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:17:37,167] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:17:44,621] INFO : VALIDATION | Step=33000/50000 | Train Loss=7.2547 | Valid Loss=7.6175 | Train Loss Unweighted=2.6361 | Valid Loss Unweighted=2.7176 | Stale=0 | ETA=126.3m +rank0 [2026-10-28 15:17:44,630] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:17:44,699] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:17:44,702] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:17:44,871] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:17:46,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:17:47,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:17:48,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:17:48,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:17:49,011] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:17:49,014] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:17:49,083] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:17:49,086] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:17:49,188] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:17:51,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:17:51,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:17:52,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:17:53,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:17:53,248] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:17:53,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:17:53,315] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:17:53,318] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:17:53,420] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:17:55,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:17:55,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:17:56,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:17:57,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:17:57,303] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:17:57,310] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:17:57,569] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:17:57,974] INFO : Plotted number of elements +rank0 [2026-10-28 15:17:59,416] INFO : Plotted jets +rank0 [2026-10-28 15:18:00,512] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:00,544] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:00,772] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:01,161] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:03,093] INFO : Plotted jets +rank0 [2026-10-28 15:18:04,210] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:04,242] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 15:18:04,447] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_33000/cld_edm_qq_pf +rank0 [2026-10-28 15:18:04,807] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:06,184] INFO : Plotted jets +rank0 [2026-10-28 15:18:07,251] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:48,270] INFO : Step 33100/50000 rank0 | Train Loss: 7.5267 | Train Loss Unweighted: 2.7300 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.4449 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1232 | Regression_eta: 0.0131 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7898 | Classification_binary: 2.4449 | Regression_cos_phi: 0.4579 | Regression_energy: 1.3503 | Regression_eta: 0.6430 | Regression_pt: 1.4223 | Regression_sin_phi: 0.4185 | LR: 3.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7263s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.2668 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3931 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5508.5 | uss_MB: 4746.5 | child_rss_MB: 3994.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:18:48,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:19:29,592] INFO : Step 33200/50000 rank0 | Train Loss: 7.1180 | Train Loss Unweighted: 2.6548 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.3807 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1188 | Regression_eta: 0.0092 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7422 | Classification_binary: 2.3807 | Regression_cos_phi: 0.4525 | Regression_energy: 1.3019 | Regression_eta: 0.4526 | Regression_pt: 1.4108 | Regression_sin_phi: 0.3774 | LR: 3.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7588s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3077 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.3882 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5508.5 | uss_MB: 4746.5 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:19:29,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:10,343] INFO : Step 33300/50000 rank0 | Train Loss: 7.1069 | Train Loss Unweighted: 2.6617 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3869 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1204 | Regression_eta: 0.0080 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8060 | Classification_binary: 2.3869 | Regression_cos_phi: 0.4382 | Regression_energy: 1.3194 | Regression_eta: 0.3936 | Regression_pt: 1.4051 | Regression_sin_phi: 0.3577 | LR: 3.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7273s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.2806 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.3853 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5508.5 | uss_MB: 4746.5 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:20:10,556] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:50,957] INFO : Step 33400/50000 rank0 | Train Loss: 8.0608 | Train Loss Unweighted: 2.7959 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4736 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1406 | Regression_eta: 0.0097 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9145 | Classification_binary: 2.4736 | Regression_cos_phi: 0.4850 | Regression_energy: 1.5411 | Regression_eta: 0.4746 | Regression_pt: 1.6421 | Regression_sin_phi: 0.5299 | LR: 3.23e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7485s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3078 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3901 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5388.5 | uss_MB: 4625.5 | child_rss_MB: 3994.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:20:51,168] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:21:31,963] INFO : Step 33500/50000 rank0 | Train Loss: 7.7721 | Train Loss Unweighted: 2.6624 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3629 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1282 | Regression_eta: 0.0153 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8123 | Classification_binary: 2.3629 | Regression_cos_phi: 0.4807 | Regression_energy: 1.4052 | Regression_eta: 0.7503 | Regression_pt: 1.4892 | Regression_sin_phi: 0.4716 | LR: 3.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7035s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1613 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0130 | Memory: rss_MB: 5388.5 | uss_MB: 4625.6 | child_rss_MB: 3991.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:21:32,176] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:12,658] INFO : Step 33600/50000 rank0 | Train Loss: 8.1198 | Train Loss Unweighted: 2.7941 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4647 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1458 | Regression_eta: 0.0129 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8506 | Classification_binary: 2.4647 | Regression_cos_phi: 0.4983 | Regression_energy: 1.5981 | Regression_eta: 0.6358 | Regression_pt: 1.6630 | Regression_sin_phi: 0.4094 | LR: 3.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7697s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1853 | max_loss: 0.0033 | max_backward: 0.2021 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.2987 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5388.5 | uss_MB: 4625.6 | child_rss_MB: 3995.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:22:12,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:54,418] INFO : Step 33700/50000 rank0 | Train Loss: 7.4886 | Train Loss Unweighted: 2.8212 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.5285 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1272 | Regression_eta: 0.0099 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7640 | Classification_binary: 2.5285 | Regression_cos_phi: 0.4268 | Regression_energy: 1.3937 | Regression_eta: 0.4844 | Regression_pt: 1.5277 | Regression_sin_phi: 0.3635 | LR: 3.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7703s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0206 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3073 | max_optimizer_step: 0.1420 | max_tensorboard_logging: 0.2974 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5388.5 | uss_MB: 4625.3 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:22:54,633] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:23:35,318] INFO : Step 33800/50000 rank0 | Train Loss: 7.7957 | Train Loss Unweighted: 2.7921 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4685 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1439 | Regression_eta: 0.0090 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8482 | Classification_binary: 2.4685 | Regression_cos_phi: 0.4039 | Regression_energy: 1.5769 | Regression_eta: 0.4435 | Regression_pt: 1.6884 | Regression_sin_phi: 0.3663 | LR: 3.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6926s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0719 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3070 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5388.5 | uss_MB: 4625.4 | child_rss_MB: 4021.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:23:35,529] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:16,273] INFO : Step 33900/50000 rank0 | Train Loss: 7.8186 | Train Loss Unweighted: 2.6774 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3708 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1318 | Regression_eta: 0.0162 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8111 | Classification_binary: 2.3708 | Regression_cos_phi: 0.4453 | Regression_energy: 1.4446 | Regression_eta: 0.7982 | Regression_pt: 1.5381 | Regression_sin_phi: 0.4104 | LR: 3.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7369s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.3037 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3913 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5388.5 | uss_MB: 4625.3 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:24:16,486] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:56,621] INFO : Step 34000/50000 rank0 | Train Loss: 8.1339 | Train Loss Unweighted: 2.7972 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4808 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1376 | Regression_eta: 0.0141 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8995 | Classification_binary: 2.4808 | Regression_cos_phi: 0.5625 | Regression_energy: 1.5077 | Regression_eta: 0.6955 | Regression_pt: 1.5660 | Regression_sin_phi: 0.4218 | LR: 3.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6755s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1043 | max_loss: 0.0035 | max_backward: 0.2025 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.3045 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5387.9 | uss_MB: 4625.0 | child_rss_MB: 3990.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:24:56,835] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:56,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 15:24:57,031] INFO : Running validation on rank0 +rank0 [2026-10-28 15:24:57,031] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:24:57,031] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:24:58,366] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 15:24:58,512] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:24:58,646] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:24:58,720] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:24:58,790] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:24:58,859] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:24:58,928] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:24:58,999] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:24:59,112] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:24:59,245] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:24:59,379] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:24:59,451] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:24:59,521] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:24:59,590] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:24:59,660] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:24:59,730] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:24:59,827] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:24:59,833] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:24:59,836] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:24:59,839] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:24:59,840] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:24:59,843] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:24:59,844] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:24:59,847] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:24:59,848] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:24:59,850] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:24:59,853] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:24:59,854] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:24:59,856] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:24:59,857] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:24:59,860] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:24:59,863] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:25:07,587] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.1339 | Valid Loss=7.5847 | Train Loss Unweighted=2.7972 | Valid Loss Unweighted=2.7077 | Stale=0 | ETA=118.9m +rank0 [2026-10-28 15:25:07,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:07,656] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:25:07,659] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:25:07,762] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:09,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:10,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:11,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:11,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:12,146] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:12,149] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:12,221] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:25:12,224] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:12,317] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:14,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:14,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:15,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:16,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:16,247] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:25:16,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:16,312] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:25:16,315] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:25:16,421] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:18,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:19,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:19,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:20,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:20,443] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:20,451] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:25:20,711] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:25:21,127] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:22,563] INFO : Plotted jets +rank0 [2026-10-28 15:25:24,259] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:24,291] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:24,589] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:24,984] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:26,378] INFO : Plotted jets +rank0 [2026-10-28 15:25:27,501] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:27,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 15:25:27,745] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_34000/cld_edm_qq_pf +rank0 [2026-10-28 15:25:28,108] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:29,494] INFO : Plotted jets +rank0 [2026-10-28 15:25:30,555] INFO : Plotted jet ratio +rank0 [2026-10-28 15:26:12,377] INFO : Step 34100/50000 rank0 | Train Loss: 7.5329 | Train Loss Unweighted: 2.7521 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4639 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1254 | Regression_eta: 0.0110 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8241 | Classification_binary: 2.4639 | Regression_cos_phi: 0.4626 | Regression_energy: 1.3741 | Regression_eta: 0.5430 | Regression_pt: 1.4480 | Regression_sin_phi: 0.4171 | LR: 3.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7530s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1854 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2375 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5599.2 | uss_MB: 4836.1 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:26:12,592] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:26:53,982] INFO : Step 34200/50000 rank0 | Train Loss: 7.8139 | Train Loss Unweighted: 2.7777 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4532 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1433 | Regression_eta: 0.0082 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7757 | Classification_binary: 2.4532 | Regression_cos_phi: 0.4468 | Regression_energy: 1.5704 | Regression_eta: 0.4017 | Regression_pt: 1.7107 | Regression_sin_phi: 0.4553 | LR: 3.04e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8173s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0611 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3086 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2895 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5599.2 | uss_MB: 4835.9 | child_rss_MB: 3991.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:26:54,199] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:27:35,069] INFO : Step 34300/50000 rank0 | Train Loss: 7.3762 | Train Loss Unweighted: 2.7042 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4327 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1165 | Regression_eta: 0.0119 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7962 | Classification_binary: 2.4327 | Regression_cos_phi: 0.4959 | Regression_energy: 1.2760 | Regression_eta: 0.5842 | Regression_pt: 1.3416 | Regression_sin_phi: 0.4497 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7711s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2120 | max_loss: 0.0033 | max_backward: 0.2863 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 0.2857 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5599.2 | uss_MB: 4838.2 | child_rss_MB: 3994.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:27:35,282] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:28:16,832] INFO : Step 34400/50000 rank0 | Train Loss: 8.0568 | Train Loss Unweighted: 2.7796 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4552 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1418 | Regression_eta: 0.0140 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7782 | Classification_binary: 2.4552 | Regression_cos_phi: 0.5007 | Regression_energy: 1.5540 | Regression_eta: 0.6901 | Regression_pt: 1.6531 | Regression_sin_phi: 0.4255 | LR: 2.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7140s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2708 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5461.3 | uss_MB: 4698.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:28:17,045] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:28:57,673] INFO : Step 34500/50000 rank0 | Train Loss: 6.9764 | Train Loss Unweighted: 2.6937 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4157 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1235 | Regression_eta: 0.0068 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7795 | Classification_binary: 2.4157 | Regression_cos_phi: 0.3641 | Regression_energy: 1.3533 | Regression_eta: 0.3327 | Regression_pt: 1.4549 | Regression_sin_phi: 0.2761 | LR: 2.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6525s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1824 | max_loss: 0.0032 | max_backward: 0.2026 | max_optimizer_step: 0.1378 | max_tensorboard_logging: 0.2968 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5461.3 | uss_MB: 4698.5 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:28:57,898] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:29:38,428] INFO : Step 34600/50000 rank0 | Train Loss: 7.9621 | Train Loss Unweighted: 2.8351 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5358 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1284 | Regression_eta: 0.0127 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9445 | Classification_binary: 2.5358 | Regression_cos_phi: 0.4668 | Regression_energy: 1.4071 | Regression_eta: 0.6228 | Regression_pt: 1.4766 | Regression_sin_phi: 0.5085 | LR: 2.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7101s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0861 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3083 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5461.3 | uss_MB: 4698.5 | child_rss_MB: 4021.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:29:38,641] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:30:18,763] INFO : Step 34700/50000 rank0 | Train Loss: 7.4332 | Train Loss Unweighted: 2.7127 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.4199 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1278 | Regression_eta: 0.0069 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7871 | Classification_binary: 2.4199 | Regression_cos_phi: 0.5628 | Regression_energy: 1.4008 | Regression_eta: 0.3386 | Regression_pt: 1.5191 | Regression_sin_phi: 0.4049 | LR: 2.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5937s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0310 | max_backward: 0.2025 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3098 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5461.3 | uss_MB: 4698.6 | child_rss_MB: 3993.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:30:18,977] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:00,858] INFO : Step 34800/50000 rank0 | Train Loss: 7.4785 | Train Loss Unweighted: 2.5928 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2955 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1285 | Regression_eta: 0.0088 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8684 | Classification_binary: 2.2955 | Regression_cos_phi: 0.5802 | Regression_energy: 1.4076 | Regression_eta: 0.4349 | Regression_pt: 1.5253 | Regression_sin_phi: 0.3666 | LR: 2.90e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.8628s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0200 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.3900 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5462.3 | uss_MB: 4701.1 | child_rss_MB: 3995.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 15:31:01,073] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:41,917] INFO : Step 34900/50000 rank0 | Train Loss: 7.8591 | Train Loss Unweighted: 2.7561 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4324 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1426 | Regression_eta: 0.0112 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7775 | Classification_binary: 2.4324 | Regression_cos_phi: 0.4808 | Regression_energy: 1.5620 | Regression_eta: 0.5490 | Regression_pt: 1.6841 | Regression_sin_phi: 0.3732 | LR: 2.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7283s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0032 | max_backward: 0.2023 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5461.3 | uss_MB: 4698.9 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:31:42,145] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:23,135] INFO : Step 35000/50000 rank0 | Train Loss: 7.7028 | Train Loss Unweighted: 2.7187 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4207 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1300 | Regression_eta: 0.0115 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7966 | Classification_binary: 2.4207 | Regression_cos_phi: 0.5441 | Regression_energy: 1.4248 | Regression_eta: 0.5654 | Regression_pt: 1.4905 | Regression_sin_phi: 0.4607 | LR: 2.85e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8462s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1042 | max_loss: 0.0034 | max_backward: 0.3926 | max_optimizer_step: 0.1341 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5460.7 | uss_MB: 4700.6 | child_rss_MB: 3999.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:32:23,348] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:23,369] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 15:32:23,547] INFO : Running validation on rank0 +rank0 [2026-10-28 15:32:23,548] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:32:23,548] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:32:24,964] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 15:32:25,111] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:32:25,244] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:32:25,319] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:32:25,390] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:32:25,459] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:32:25,529] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:32:25,598] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:32:25,712] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:32:25,843] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:32:25,975] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:32:26,047] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:32:26,117] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:32:26,186] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:32:26,256] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:32:26,327] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:32:26,427] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:32:26,433] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:32:26,436] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:32:26,439] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:32:26,440] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:32:26,442] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:32:26,444] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:32:26,446] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:32:26,449] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:32:26,452] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:32:26,453] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:32:26,455] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:32:26,458] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:32:26,459] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:32:26,461] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:32:26,463] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:32:34,169] INFO : VALIDATION | Step=35000/50000 | Train Loss=7.7028 | Valid Loss=7.5819 | Train Loss Unweighted=2.7187 | Valid Loss Unweighted=2.6987 | Stale=0 | ETA=111.4m +rank0 [2026-10-28 15:32:34,179] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:34,254] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:32:34,258] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:32:34,431] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:36,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:37,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:38,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:38,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:38,867] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:38,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:38,966] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:32:38,976] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:39,159] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:41,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:41,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:42,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:43,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:43,538] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:43,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:43,620] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:32:43,623] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:32:43,804] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:45,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:46,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:47,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:47,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:47,809] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:47,817] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:32:48,074] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:32:48,481] INFO : Plotted number of elements +rank0 [2026-10-28 15:32:49,930] INFO : Plotted jets +rank0 [2026-10-28 15:32:51,032] INFO : Plotted jet ratio +rank0 [2026-10-28 15:32:51,066] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:51,292] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:51,685] INFO : Plotted number of elements +rank0 [2026-10-28 15:32:53,625] INFO : Plotted jets +rank0 [2026-10-28 15:32:54,745] INFO : Plotted jet ratio +rank0 [2026-10-28 15:32:54,779] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 15:32:54,989] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_35000/cld_edm_qq_pf +rank0 [2026-10-28 15:32:55,353] INFO : Plotted number of elements +rank0 [2026-10-28 15:32:56,732] INFO : Plotted jets +rank0 [2026-10-28 15:32:57,803] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:39,450] INFO : Step 35100/50000 rank0 | Train Loss: 7.6070 | Train Loss Unweighted: 2.6246 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3332 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1242 | Regression_eta: 0.0143 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8692 | Classification_binary: 2.3332 | Regression_cos_phi: 0.5004 | Regression_energy: 1.3608 | Regression_eta: 0.7024 | Regression_pt: 1.4499 | Regression_sin_phi: 0.3911 | LR: 2.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.0018s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0936 | max_forward: 0.1044 | max_loss: 0.0035 | max_backward: 0.3943 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.3961 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5641.5 | uss_MB: 4884.5 | child_rss_MB: 4002.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:33:39,664] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 2% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:34:19,661] INFO : Step 35200/50000 rank0 | Train Loss: 6.8227 | Train Loss Unweighted: 2.5979 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.3362 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1161 | Regression_eta: 0.0095 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7395 | Classification_binary: 2.3362 | Regression_cos_phi: 0.3644 | Regression_energy: 1.2720 | Regression_eta: 0.4676 | Regression_pt: 1.3170 | Regression_sin_phi: 0.3259 | LR: 2.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6228s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0033 | max_backward: 0.2025 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5641.5 | uss_MB: 4882.4 | child_rss_MB: 4000.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:34:19,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:01,017] INFO : Step 35300/50000 rank0 | Train Loss: 7.1455 | Train Loss Unweighted: 2.6649 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3932 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1179 | Regression_eta: 0.0090 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7866 | Classification_binary: 2.3932 | Regression_cos_phi: 0.4653 | Regression_energy: 1.2917 | Regression_eta: 0.4430 | Regression_pt: 1.3815 | Regression_sin_phi: 0.3842 | LR: 2.79e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7563s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1826 | max_loss: 0.0037 | max_backward: 0.2025 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.2976 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5641.5 | uss_MB: 4882.4 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:35:01,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:42,787] INFO : Step 35400/50000 rank0 | Train Loss: 7.1655 | Train Loss Unweighted: 2.6257 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3535 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1184 | Regression_eta: 0.0084 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8892 | Classification_binary: 2.3535 | Regression_cos_phi: 0.4371 | Regression_energy: 1.2971 | Regression_eta: 0.4145 | Regression_pt: 1.3609 | Regression_sin_phi: 0.4132 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7660s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3042 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.4059 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5529.2 | uss_MB: 4769.4 | child_rss_MB: 3996.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:35:43,001] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:36:23,727] INFO : Step 35500/50000 rank0 | Train Loss: 7.6174 | Train Loss Unweighted: 2.7660 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4689 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1311 | Regression_eta: 0.0098 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8402 | Classification_binary: 2.4689 | Regression_cos_phi: 0.4731 | Regression_energy: 1.4369 | Regression_eta: 0.4807 | Regression_pt: 1.4946 | Regression_sin_phi: 0.4231 | LR: 2.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7564s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.2101 | max_loss: 0.0034 | max_backward: 0.2868 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5529.2 | uss_MB: 4772.9 | child_rss_MB: 3995.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 15:36:23,940] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:05,210] INFO : Step 35600/50000 rank0 | Train Loss: 7.5924 | Train Loss Unweighted: 2.6903 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3821 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1348 | Regression_eta: 0.0109 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7928 | Classification_binary: 2.3821 | Regression_cos_phi: 0.3767 | Regression_energy: 1.4765 | Regression_eta: 0.5382 | Regression_pt: 1.5986 | Regression_sin_phi: 0.4276 | LR: 2.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7339s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0879 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.3104 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5529.2 | uss_MB: 4771.0 | child_rss_MB: 4028.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:37:05,426] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:46,127] INFO : Step 35700/50000 rank0 | Train Loss: 7.4640 | Train Loss Unweighted: 2.7411 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4584 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1229 | Regression_eta: 0.0073 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7987 | Classification_binary: 2.4584 | Regression_cos_phi: 0.5640 | Regression_energy: 1.3463 | Regression_eta: 0.3594 | Regression_pt: 1.4306 | Regression_sin_phi: 0.5066 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7552s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0034 | max_backward: 0.3108 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3810 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5529.2 | uss_MB: 4771.3 | child_rss_MB: 4001.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:37:46,339] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:38:27,035] INFO : Step 35800/50000 rank0 | Train Loss: 7.4530 | Train Loss Unweighted: 2.6901 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3893 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1338 | Regression_eta: 0.0079 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8189 | Classification_binary: 2.3893 | Regression_cos_phi: 0.4781 | Regression_energy: 1.4661 | Regression_eta: 0.3897 | Regression_pt: 1.5444 | Regression_sin_phi: 0.3665 | LR: 2.68e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7834s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1044 | max_loss: 0.0034 | max_backward: 0.3855 | max_optimizer_step: 0.1399 | max_tensorboard_logging: 0.3330 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5529.2 | uss_MB: 4773.4 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:38:27,246] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:07,221] INFO : Step 35900/50000 rank0 | Train Loss: 7.1536 | Train Loss Unweighted: 2.6729 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3892 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1245 | Regression_eta: 0.0069 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8754 | Classification_binary: 2.3892 | Regression_cos_phi: 0.3500 | Regression_energy: 1.3645 | Regression_eta: 0.3386 | Regression_pt: 1.4721 | Regression_sin_phi: 0.3640 | LR: 2.65e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6332s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0907 | max_loss: 0.0033 | max_backward: 0.2022 | max_optimizer_step: 0.2233 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5529.2 | uss_MB: 4771.3 | child_rss_MB: 3998.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:39:07,437] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:47,648] INFO : Step 36000/50000 rank0 | Train Loss: 8.2734 | Train Loss Unweighted: 2.8643 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5492 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1359 | Regression_eta: 0.0184 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8597 | Classification_binary: 2.5492 | Regression_cos_phi: 0.5455 | Regression_energy: 1.4886 | Regression_eta: 0.9042 | Regression_pt: 1.5417 | Regression_sin_phi: 0.3845 | LR: 2.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6587s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2106 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.3959 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5528.6 | uss_MB: 4769.4 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:39:47,863] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:47,908] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 15:39:48,089] INFO : Running validation on rank0 +rank0 [2026-10-28 15:39:48,090] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:39:48,090] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:39:49,417] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 15:39:49,564] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:39:49,696] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:39:49,770] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:39:49,839] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:39:49,908] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:39:49,977] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:39:50,046] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:39:50,160] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:39:50,294] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:39:50,426] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:39:50,497] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:39:50,566] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:39:50,634] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:39:50,703] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:39:50,772] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:39:50,872] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:39:50,878] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:39:50,881] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:39:50,882] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:39:50,885] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:39:50,887] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:39:50,889] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:39:50,891] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:39:50,894] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:39:50,895] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:39:50,897] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:39:50,900] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:39:50,901] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:39:50,904] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:39:50,906] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:39:50,909] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:39:58,521] INFO : VALIDATION | Step=36000/50000 | Train Loss=8.2734 | Valid Loss=7.5099 | Train Loss Unweighted=2.8643 | Valid Loss Unweighted=2.6828 | Stale=0 | ETA=104.0m +rank0 [2026-10-28 15:39:58,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:39:58,586] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:39:58,590] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:39:58,764] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:00,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:01,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:01,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:02,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:02,806] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:02,809] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:02,875] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:40:02,878] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:02,975] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:04,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:05,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:06,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:06,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:07,010] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:07,013] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:07,084] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:40:07,087] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:40:07,186] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:09,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:09,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:10,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:11,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:11,164] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:11,173] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:40:11,426] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:40:11,829] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:13,263] INFO : Plotted jets +rank0 [2026-10-28 15:40:14,358] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:14,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:14,616] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:15,003] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:16,942] INFO : Plotted jets +rank0 [2026-10-28 15:40:18,059] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:18,092] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 15:40:18,298] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_36000/cld_edm_qq_pf +rank0 [2026-10-28 15:40:18,661] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:20,039] INFO : Plotted jets +rank0 [2026-10-28 15:40:21,112] INFO : Plotted jet ratio +rank0 [2026-10-28 15:41:03,045] INFO : Step 36100/50000 rank0 | Train Loss: 7.1912 | Train Loss Unweighted: 2.7191 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4420 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1222 | Regression_eta: 0.0092 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8289 | Classification_binary: 2.4420 | Regression_cos_phi: 0.3668 | Regression_energy: 1.3386 | Regression_eta: 0.4547 | Regression_pt: 1.4014 | Regression_sin_phi: 0.3588 | LR: 2.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7572s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5736.2 | uss_MB: 4976.1 | child_rss_MB: 3998.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:41:03,260] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:41:43,455] INFO : Step 36200/50000 rank0 | Train Loss: 6.9320 | Train Loss Unweighted: 2.5627 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.2922 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1171 | Regression_eta: 0.0069 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8457 | Classification_binary: 2.2922 | Regression_cos_phi: 0.3791 | Regression_energy: 1.2836 | Regression_eta: 0.3396 | Regression_pt: 1.3984 | Regression_sin_phi: 0.3935 | LR: 2.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6833s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.2286 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5736.2 | uss_MB: 4976.1 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:41:43,666] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:42:24,804] INFO : Step 36300/50000 rank0 | Train Loss: 7.5767 | Train Loss Unweighted: 2.6947 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3701 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1466 | Regression_eta: 0.0070 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7749 | Classification_binary: 2.3701 | Regression_cos_phi: 0.3863 | Regression_energy: 1.6064 | Regression_eta: 0.3421 | Regression_pt: 1.7117 | Regression_sin_phi: 0.3852 | LR: 2.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3090 | max_optimizer_step: 0.1432 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5736.2 | uss_MB: 4976.1 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:42:25,018] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:06,128] INFO : Step 36400/50000 rank0 | Train Loss: 7.1193 | Train Loss Unweighted: 2.6120 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3425 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1160 | Regression_eta: 0.0089 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8668 | Classification_binary: 2.3425 | Regression_cos_phi: 0.4585 | Regression_energy: 1.2706 | Regression_eta: 0.4387 | Regression_pt: 1.3590 | Regression_sin_phi: 0.3831 | LR: 2.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8124s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0109 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.3105 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.4086 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5604.2 | uss_MB: 4841.4 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:43:06,341] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:46,922] INFO : Step 36500/50000 rank0 | Train Loss: 7.6218 | Train Loss Unweighted: 2.6561 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3535 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1325 | Regression_eta: 0.0108 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8798 | Classification_binary: 2.3535 | Regression_cos_phi: 0.4333 | Regression_energy: 1.4515 | Regression_eta: 0.5305 | Regression_pt: 1.5242 | Regression_sin_phi: 0.4491 | LR: 2.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9065s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0307 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3939 | max_optimizer_step: 0.1436 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5604.2 | uss_MB: 4844.8 | child_rss_MB: 3996.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:43:47,139] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:44:27,277] INFO : Step 36600/50000 rank0 | Train Loss: 7.9635 | Train Loss Unweighted: 2.7214 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4120 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1345 | Regression_eta: 0.0129 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8571 | Classification_binary: 2.4120 | Regression_cos_phi: 0.5791 | Regression_energy: 1.4733 | Regression_eta: 0.6365 | Regression_pt: 1.5330 | Regression_sin_phi: 0.4725 | LR: 2.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6316s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0033 | max_backward: 0.2022 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5604.2 | uss_MB: 4843.3 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:44:27,491] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:08,462] INFO : Step 36700/50000 rank0 | Train Loss: 8.4009 | Train Loss Unweighted: 2.8552 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5246 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1431 | Regression_eta: 0.0174 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8535 | Classification_binary: 2.5246 | Regression_cos_phi: 0.4921 | Regression_energy: 1.5678 | Regression_eta: 0.8559 | Regression_pt: 1.6487 | Regression_sin_phi: 0.4583 | LR: 2.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.1019 | max_backward: 0.2026 | max_optimizer_step: 0.1410 | max_tensorboard_logging: 0.2998 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5605.2 | uss_MB: 4843.5 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:45:08,675] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:50,442] INFO : Step 36800/50000 rank0 | Train Loss: 7.6975 | Train Loss Unweighted: 2.8132 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.5192 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1290 | Regression_eta: 0.0089 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8190 | Classification_binary: 2.5192 | Regression_cos_phi: 0.5984 | Regression_energy: 1.4138 | Regression_eta: 0.4357 | Regression_pt: 1.4757 | Regression_sin_phi: 0.4357 | LR: 2.46e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9258s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0417 | max_forward: 0.1038 | max_loss: 0.0033 | max_backward: 0.3930 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.3963 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 5605.2 | uss_MB: 4845.7 | child_rss_MB: 3998.7 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 15:45:50,656] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:46:31,508] INFO : Step 36900/50000 rank0 | Train Loss: 7.5002 | Train Loss Unweighted: 2.7044 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4188 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1235 | Regression_eta: 0.0115 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8139 | Classification_binary: 2.4188 | Regression_cos_phi: 0.5367 | Regression_energy: 1.3529 | Regression_eta: 0.5636 | Regression_pt: 1.4319 | Regression_sin_phi: 0.3824 | LR: 2.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7662s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1854 | max_loss: 0.0035 | max_backward: 0.2024 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5605.2 | uss_MB: 4843.6 | child_rss_MB: 4000.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:46:31,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:12,436] INFO : Step 37000/50000 rank0 | Train Loss: 7.6896 | Train Loss Unweighted: 2.7126 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4072 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1345 | Regression_eta: 0.0114 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7726 | Classification_binary: 2.4072 | Regression_cos_phi: 0.4795 | Regression_energy: 1.4743 | Regression_eta: 0.5589 | Regression_pt: 1.5460 | Regression_sin_phi: 0.4511 | LR: 2.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.9567s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1987 | max_loss: 0.0033 | max_backward: 0.3922 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.2924 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 5604.0 | uss_MB: 4843.3 | child_rss_MB: 4000.1 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:47:12,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:12,668] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 15:47:12,947] INFO : Running validation on rank0 +rank0 [2026-10-28 15:47:12,947] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:47:12,948] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:47:14,390] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 15:47:14,536] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:47:14,668] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:47:14,742] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:47:14,811] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:47:14,880] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:47:14,949] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:47:15,018] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:47:15,131] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:47:15,263] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:47:15,394] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:47:15,466] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:47:15,535] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:47:15,604] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:47:15,672] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:47:15,742] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:47:15,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:47:15,850] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:47:15,853] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:47:15,856] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:47:15,857] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:47:15,859] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:47:15,861] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:47:15,863] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:47:15,864] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:47:15,867] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:47:15,870] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:47:15,871] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:47:15,873] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:47:15,876] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:47:15,877] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:47:15,880] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:47:23,634] INFO : VALIDATION | Step=37000/50000 | Train Loss=7.6896 | Valid Loss=7.5400 | Train Loss Unweighted=2.7126 | Valid Loss Unweighted=2.6929 | Stale=1 | ETA=96.6m +rank0 [2026-10-28 15:47:23,644] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:23,732] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:47:23,736] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:47:23,950] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:25,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:26,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:27,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:28,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:28,183] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:28,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:28,243] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:47:28,246] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:28,355] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:30,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:30,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:31,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:32,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:32,367] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:32,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:32,441] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:47:32,444] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:47:32,544] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:34,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:34,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:35,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:36,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:36,467] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:47:36,476] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:47:36,738] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:47:37,144] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:38,585] INFO : Plotted jets +rank0 [2026-10-28 15:47:39,680] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:39,712] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:39,942] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:40,334] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:42,283] INFO : Plotted jets +rank0 [2026-10-28 15:47:43,410] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:43,444] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 15:47:43,651] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_37000/cld_edm_qq_pf +rank0 [2026-10-28 15:47:44,011] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:45,617] INFO : Plotted jets +rank0 [2026-10-28 15:47:46,691] INFO : Plotted jet ratio +rank0 [2026-10-28 15:48:27,765] INFO : Step 37100/50000 rank0 | Train Loss: 7.5056 | Train Loss Unweighted: 2.7824 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4934 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1251 | Regression_eta: 0.0073 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8272 | Classification_binary: 2.4934 | Regression_cos_phi: 0.5464 | Regression_energy: 1.3705 | Regression_eta: 0.3587 | Regression_pt: 1.4926 | Regression_sin_phi: 0.4169 | LR: 2.40e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6667s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0814 | max_backward: 0.2029 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5812.9 | uss_MB: 5050.8 | child_rss_MB: 3999.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:48:27,978] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:09,103] INFO : Step 37200/50000 rank0 | Train Loss: 7.2977 | Train Loss Unweighted: 2.5856 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.3135 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1166 | Regression_eta: 0.0115 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9662 | Classification_binary: 2.3135 | Regression_cos_phi: 0.4549 | Regression_energy: 1.2774 | Regression_eta: 0.5659 | Regression_pt: 1.3239 | Regression_sin_phi: 0.3959 | LR: 2.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6228s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0654 | max_backward: 0.2029 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5812.9 | uss_MB: 5050.8 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:49:09,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:50,309] INFO : Step 37300/50000 rank0 | Train Loss: 7.0843 | Train Loss Unweighted: 2.6577 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3738 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1256 | Regression_eta: 0.0086 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7711 | Classification_binary: 2.3738 | Regression_cos_phi: 0.3363 | Regression_energy: 1.3766 | Regression_eta: 0.4212 | Regression_pt: 1.4769 | Regression_sin_phi: 0.3285 | LR: 2.36e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0228 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3115 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.4060 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 5812.9 | uss_MB: 5050.8 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:49:50,521] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:50:31,240] INFO : Step 37400/50000 rank0 | Train Loss: 7.8018 | Train Loss Unweighted: 2.5691 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.2703 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1265 | Regression_eta: 0.0260 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7672 | Classification_binary: 2.2703 | Regression_cos_phi: 0.3586 | Regression_energy: 1.3856 | Regression_eta: 1.2767 | Regression_pt: 1.4372 | Regression_sin_phi: 0.3061 | LR: 2.34e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0874 | max_backward: 0.2027 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3526 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5679.6 | uss_MB: 4916.3 | child_rss_MB: 4000.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:50:31,454] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:51:13,364] INFO : Step 37500/50000 rank0 | Train Loss: 8.2378 | Train Loss Unweighted: 2.8042 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4701 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1472 | Regression_eta: 0.0102 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8889 | Classification_binary: 2.4701 | Regression_cos_phi: 0.5466 | Regression_energy: 1.6133 | Regression_eta: 0.5029 | Regression_pt: 1.6965 | Regression_sin_phi: 0.5195 | LR: 2.32e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.7955s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0785 | max_loss: 0.0037 | max_backward: 0.3101 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4010 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5680.6 | uss_MB: 4917.4 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:51:13,577] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:51:55,509] INFO : Step 37600/50000 rank0 | Train Loss: 7.1560 | Train Loss Unweighted: 2.6325 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3552 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1196 | Regression_eta: 0.0073 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7977 | Classification_binary: 2.3552 | Regression_cos_phi: 0.5509 | Regression_energy: 1.3108 | Regression_eta: 0.3567 | Regression_pt: 1.4378 | Regression_sin_phi: 0.3469 | LR: 2.30e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7320s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2479 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.3941 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5680.6 | uss_MB: 4917.7 | child_rss_MB: 3998.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:51:55,722] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:52:37,053] INFO : Step 37700/50000 rank0 | Train Loss: 7.2398 | Train Loss Unweighted: 2.5846 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.2939 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1279 | Regression_eta: 0.0085 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7781 | Classification_binary: 2.2939 | Regression_cos_phi: 0.4677 | Regression_energy: 1.4019 | Regression_eta: 0.4188 | Regression_pt: 1.4961 | Regression_sin_phi: 0.3833 | LR: 2.28e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6899s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1707 | max_loss: 0.0034 | max_backward: 0.2034 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.2805 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5680.6 | uss_MB: 4917.9 | child_rss_MB: 4000.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:52:37,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:18,803] INFO : Step 37800/50000 rank0 | Train Loss: 7.9413 | Train Loss Unweighted: 2.7216 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4165 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1356 | Regression_eta: 0.0162 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8250 | Classification_binary: 2.4165 | Regression_cos_phi: 0.5321 | Regression_energy: 1.4853 | Regression_eta: 0.7951 | Regression_pt: 1.4517 | Regression_sin_phi: 0.4355 | LR: 2.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7626s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3030 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.3967 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 5680.6 | uss_MB: 4918.0 | child_rss_MB: 4000.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:53:19,019] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:59,792] INFO : Step 37900/50000 rank0 | Train Loss: 7.6456 | Train Loss Unweighted: 2.6682 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3785 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1241 | Regression_eta: 0.0175 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7853 | Classification_binary: 2.3785 | Regression_cos_phi: 0.4732 | Regression_energy: 1.3597 | Regression_eta: 0.8623 | Regression_pt: 1.4236 | Regression_sin_phi: 0.3632 | LR: 2.24e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6336s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0438 | max_backward: 0.2027 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3151 | max_tensorboard_flush: 0.0166 | Memory: rss_MB: 5680.6 | uss_MB: 4918.0 | child_rss_MB: 3997.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:54:00,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:41,654] INFO : Step 38000/50000 rank0 | Train Loss: 7.0918 | Train Loss Unweighted: 2.7374 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.4624 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1201 | Regression_eta: 0.0067 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7422 | Classification_binary: 2.4624 | Regression_cos_phi: 0.4054 | Regression_energy: 1.3161 | Regression_eta: 0.3279 | Regression_pt: 1.4413 | Regression_sin_phi: 0.3966 | LR: 2.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8734s | Phase Times: max_pre_step_barrier: 0.1043 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3102 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.3933 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5678.9 | uss_MB: 4916.0 | child_rss_MB: 4000.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:54:41,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:41,964] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 15:54:42,144] INFO : Running validation on rank0 +rank0 [2026-10-28 15:54:42,144] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:54:42,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:54:43,454] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 15:54:43,600] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:54:43,733] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:54:43,808] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:54:43,876] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:54:43,945] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:54:44,014] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:54:44,084] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:54:44,198] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:54:44,330] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:54:44,462] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:54:44,533] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:54:44,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:54:44,669] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:54:44,739] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:54:44,809] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:54:44,909] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:54:44,915] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:54:44,918] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:54:44,921] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:54:44,922] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:54:44,924] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:54:44,926] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:54:44,928] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:54:44,929] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:54:44,932] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:54:44,935] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:54:44,935] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:54:44,938] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:54:44,941] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:54:44,942] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:54:44,945] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:54:52,403] INFO : VALIDATION | Step=38000/50000 | Train Loss=7.0918 | Valid Loss=7.4444 | Train Loss Unweighted=2.7374 | Valid Loss Unweighted=2.6633 | Stale=0 | ETA=89.2m +rank0 [2026-10-28 15:54:52,413] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:52,479] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:54:52,482] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 15:54:52,656] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:54,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:55,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:55,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:56,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:56,841] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:54:56,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:56,908] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:54:56,911] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:57,021] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:59,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:59,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:55:00,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:55:00,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:55:01,046] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:55:01,049] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:55:01,110] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 15:55:01,112] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 15:55:01,225] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:55:03,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:55:03,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:55:04,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:55:04,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:55:05,139] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:55:05,147] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:55:05,405] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ttbar_pf +rank0 [2026-10-28 15:55:05,809] INFO : Plotted number of elements +rank0 [2026-10-28 15:55:07,254] INFO : Plotted jets +rank0 [2026-10-28 15:55:08,347] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:08,379] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:55:08,605] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 15:55:08,996] INFO : Plotted number of elements +rank0 [2026-10-28 15:55:10,946] INFO : Plotted jets +rank0 [2026-10-28 15:55:12,074] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:12,106] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 15:55:12,317] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_38000/cld_edm_qq_pf +rank0 [2026-10-28 15:55:12,678] INFO : Plotted number of elements +rank0 [2026-10-28 15:55:14,058] INFO : Plotted jets +rank0 [2026-10-28 15:55:15,133] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:55,883] INFO : Step 38100/50000 rank0 | Train Loss: 7.4852 | Train Loss Unweighted: 2.6928 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3988 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1296 | Regression_eta: 0.0113 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8044 | Classification_binary: 2.3988 | Regression_cos_phi: 0.4349 | Regression_energy: 1.4204 | Regression_eta: 0.5554 | Regression_pt: 1.4791 | Regression_sin_phi: 0.3921 | LR: 2.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6589s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.3702 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5881.2 | uss_MB: 5120.1 | child_rss_MB: 4000.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:55:56,095] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:56:37,253] INFO : Step 38200/50000 rank0 | Train Loss: 7.6688 | Train Loss Unweighted: 2.7372 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.4314 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1346 | Regression_eta: 0.0093 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8020 | Classification_binary: 2.4314 | Regression_cos_phi: 0.4821 | Regression_energy: 1.4745 | Regression_eta: 0.4571 | Regression_pt: 1.5656 | Regression_sin_phi: 0.4561 | LR: 2.18e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6287s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0685 | max_backward: 0.2034 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.2924 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5881.2 | uss_MB: 5120.2 | child_rss_MB: 3997.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:56:37,467] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:57:17,105] INFO : Step 38300/50000 rank0 | Train Loss: 7.5861 | Train Loss Unweighted: 2.6993 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.4139 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1220 | Regression_eta: 0.0105 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7757 | Classification_binary: 2.4139 | Regression_cos_phi: 0.5521 | Regression_energy: 1.3366 | Regression_eta: 0.5154 | Regression_pt: 1.4332 | Regression_sin_phi: 0.5591 | LR: 2.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1062 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3111 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5881.2 | uss_MB: 5120.0 | child_rss_MB: 3995.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:57:17,320] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:57:57,954] INFO : Step 38400/50000 rank0 | Train Loss: 7.0001 | Train Loss Unweighted: 2.6278 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3581 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1161 | Regression_eta: 0.0096 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7776 | Classification_binary: 2.3581 | Regression_cos_phi: 0.3480 | Regression_energy: 1.2717 | Regression_eta: 0.4702 | Regression_pt: 1.3965 | Regression_sin_phi: 0.3779 | LR: 2.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7657s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2778 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3982 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5751.1 | uss_MB: 4989.6 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:57:58,166] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:58:39,103] INFO : Step 38500/50000 rank0 | Train Loss: 7.5890 | Train Loss Unweighted: 2.7326 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4278 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1346 | Regression_eta: 0.0095 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 2.4278 | Regression_cos_phi: 0.4196 | Regression_energy: 1.4747 | Regression_eta: 0.4673 | Regression_pt: 1.5657 | Regression_sin_phi: 0.4203 | LR: 2.12e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7444s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0033 | max_backward: 0.3110 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5751.1 | uss_MB: 4990.7 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:58:39,315] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:59:19,931] INFO : Step 38600/50000 rank0 | Train Loss: 7.0463 | Train Loss Unweighted: 2.6368 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3701 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1149 | Regression_eta: 0.0087 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8043 | Classification_binary: 2.3701 | Regression_cos_phi: 0.4542 | Regression_energy: 1.2591 | Regression_eta: 0.4255 | Regression_pt: 1.3599 | Regression_sin_phi: 0.3732 | LR: 2.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6931s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2179 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5751.1 | uss_MB: 4990.9 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:59:20,143] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:00,003] INFO : Step 38700/50000 rank0 | Train Loss: 7.2538 | Train Loss Unweighted: 2.6871 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.4079 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1193 | Regression_eta: 0.0119 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7689 | Classification_binary: 2.4079 | Regression_cos_phi: 0.3422 | Regression_energy: 1.3072 | Regression_eta: 0.5863 | Regression_pt: 1.4421 | Regression_sin_phi: 0.3991 | LR: 2.09e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5735s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2867 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5751.1 | uss_MB: 4991.0 | child_rss_MB: 3997.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:00:00,215] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:41,690] INFO : Step 38800/50000 rank0 | Train Loss: 7.3055 | Train Loss Unweighted: 2.6311 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3468 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1206 | Regression_eta: 0.0128 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8226 | Classification_binary: 2.3468 | Regression_cos_phi: 0.3684 | Regression_energy: 1.3211 | Regression_eta: 0.6313 | Regression_pt: 1.4681 | Regression_sin_phi: 0.3471 | LR: 2.07e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7120s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2550 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5751.1 | uss_MB: 4991.0 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:00:41,901] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:01:23,493] INFO : Step 38900/50000 rank0 | Train Loss: 7.2524 | Train Loss Unweighted: 2.6470 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3575 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1274 | Regression_eta: 0.0082 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8256 | Classification_binary: 2.3575 | Regression_cos_phi: 0.3620 | Regression_energy: 1.3957 | Regression_eta: 0.4053 | Regression_pt: 1.4918 | Regression_sin_phi: 0.4146 | LR: 2.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6649s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0046 | max_forward: 0.0770 | max_loss: 0.0034 | max_backward: 0.3102 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.2805 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5751.1 | uss_MB: 4991.1 | child_rss_MB: 4027.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:01:23,707] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:02:05,452] INFO : Step 39000/50000 rank0 | Train Loss: 6.8701 | Train Loss Unweighted: 2.5289 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.2705 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1099 | Regression_eta: 0.0097 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8401 | Classification_binary: 2.2705 | Regression_cos_phi: 0.4200 | Regression_energy: 1.2041 | Regression_eta: 0.4762 | Regression_pt: 1.3091 | Regression_sin_phi: 0.3501 | LR: 2.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7726s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.2103 | max_loss: 0.0036 | max_backward: 0.2866 | max_optimizer_step: 0.1420 | max_tensorboard_logging: 0.2865 | max_tensorboard_flush: 0.0081 | Memory: rss_MB: 5750.4 | uss_MB: 4991.7 | child_rss_MB: 3993.3 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:02:05,665] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:02:05,684] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 16:02:05,859] INFO : Running validation on rank0 +rank0 [2026-10-28 16:02:05,859] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:02:05,859] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:02:07,173] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 16:02:07,321] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:02:07,453] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:02:07,527] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:02:07,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:02:07,666] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:02:07,736] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:02:07,805] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:02:07,918] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:02:08,050] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:02:08,182] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:02:08,253] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:02:08,322] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:02:08,391] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:02:08,461] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:02:08,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:02:08,627] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:02:08,633] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:02:08,636] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:02:08,639] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:02:08,640] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:02:08,643] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:02:08,645] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:02:08,646] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:02:08,649] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:02:08,652] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:02:08,653] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:02:08,655] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:02:08,658] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:02:08,660] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:02:08,661] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:02:08,664] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:02:16,230] INFO : VALIDATION | Step=39000/50000 | Train Loss=6.8701 | Valid Loss=7.4523 | Train Loss Unweighted=2.5289 | Valid Loss Unweighted=2.6673 | Stale=1 | ETA=81.7m +rank0 [2026-10-28 16:02:16,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:16,296] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:02:16,299] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:02:16,474] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:02:18,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:02:19,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:02:19,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:02:20,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:02:20,815] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:02:20,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:20,874] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:02:20,877] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:20,996] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:02:23,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:02:23,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:02:24,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:02:24,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:02:25,038] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:02:25,041] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:25,096] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:02:25,099] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:02:25,224] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:02:26,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:02:27,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:02:28,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:02:28,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:02:29,146] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:02:29,154] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:02:29,419] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:02:29,841] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:31,286] INFO : Plotted jets +rank0 [2026-10-28 16:02:32,387] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:32,418] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:32,648] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:33,043] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:35,032] INFO : Plotted jets +rank0 [2026-10-28 16:02:36,157] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:36,190] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 16:02:36,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_39000/cld_edm_qq_pf +rank0 [2026-10-28 16:02:36,763] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:38,144] INFO : Plotted jets +rank0 [2026-10-28 16:02:39,213] INFO : Plotted jet ratio +rank0 [2026-10-28 16:03:20,346] INFO : Step 39100/50000 rank0 | Train Loss: 7.4023 | Train Loss Unweighted: 2.6551 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3667 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1251 | Regression_eta: 0.0099 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8768 | Classification_binary: 2.3667 | Regression_cos_phi: 0.4153 | Regression_energy: 1.3706 | Regression_eta: 0.4859 | Regression_pt: 1.4620 | Regression_sin_phi: 0.4250 | LR: 2.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7490s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1048 | max_loss: 0.0036 | max_backward: 0.2868 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3106 | max_tensorboard_flush: 0.0148 | Memory: rss_MB: 5956.4 | uss_MB: 5197.1 | child_rss_MB: 3992.4 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:03:20,559] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:00,990] INFO : Step 39200/50000 rank0 | Train Loss: 7.6139 | Train Loss Unweighted: 2.7196 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.4241 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1280 | Regression_eta: 0.0122 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8369 | Classification_binary: 2.4241 | Regression_cos_phi: 0.4937 | Regression_energy: 1.4024 | Regression_eta: 0.6010 | Regression_pt: 1.4921 | Regression_sin_phi: 0.3637 | LR: 2.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7094s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0014 | max_forward: 0.0771 | max_loss: 0.0035 | max_backward: 0.2431 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.3806 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5956.4 | uss_MB: 5195.1 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:04:01,202] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:42,599] INFO : Step 39300/50000 rank0 | Train Loss: 7.6549 | Train Loss Unweighted: 2.6411 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3272 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1395 | Regression_eta: 0.0089 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8008 | Classification_binary: 2.3272 | Regression_cos_phi: 0.5089 | Regression_energy: 1.5282 | Regression_eta: 0.4397 | Regression_pt: 1.6062 | Regression_sin_phi: 0.4439 | LR: 1.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6589s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1017 | max_loss: 0.0037 | max_backward: 0.2031 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5956.4 | uss_MB: 5195.1 | child_rss_MB: 3991.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:04:42,814] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:05:24,415] INFO : Step 39400/50000 rank0 | Train Loss: 6.9390 | Train Loss Unweighted: 2.6161 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.3452 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1191 | Regression_eta: 0.0096 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7551 | Classification_binary: 2.3452 | Regression_cos_phi: 0.3388 | Regression_energy: 1.3044 | Regression_eta: 0.4726 | Regression_pt: 1.3887 | Regression_sin_phi: 0.3341 | LR: 1.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9139s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3075 | max_optimizer_step: 0.1428 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5956.4 | uss_MB: 5195.1 | child_rss_MB: 3994.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:05:24,626] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:05,831] INFO : Step 39500/50000 rank0 | Train Loss: 7.2029 | Train Loss Unweighted: 2.6354 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.3512 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1278 | Regression_eta: 0.0091 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7484 | Classification_binary: 2.3512 | Regression_cos_phi: 0.4819 | Regression_energy: 1.4001 | Regression_eta: 0.4470 | Regression_pt: 1.4240 | Regression_sin_phi: 0.3502 | LR: 1.94e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.7741s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3080 | max_optimizer_step: 0.1456 | max_tensorboard_logging: 0.3830 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5826.7 | uss_MB: 5066.4 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:06:06,044] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:46,554] INFO : Step 39600/50000 rank0 | Train Loss: 7.4494 | Train Loss Unweighted: 2.6677 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3763 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1268 | Regression_eta: 0.0099 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7707 | Classification_binary: 2.3763 | Regression_cos_phi: 0.4801 | Regression_energy: 1.3893 | Regression_eta: 0.4865 | Regression_pt: 1.4877 | Regression_sin_phi: 0.4588 | LR: 1.93e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8199s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1041 | max_loss: 0.0034 | max_backward: 0.3932 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3418 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5826.7 | uss_MB: 5068.8 | child_rss_MB: 3998.4 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:06:46,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:07:28,572] INFO : Step 39700/50000 rank0 | Train Loss: 7.7855 | Train Loss Unweighted: 2.6558 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3532 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1324 | Regression_eta: 0.0163 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8073 | Classification_binary: 2.3532 | Regression_cos_phi: 0.5027 | Regression_energy: 1.4511 | Regression_eta: 0.8008 | Regression_pt: 1.4756 | Regression_sin_phi: 0.3948 | LR: 1.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3095 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.3683 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5826.7 | uss_MB: 5066.8 | child_rss_MB: 3996.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:07:28,782] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:10,058] INFO : Step 39800/50000 rank0 | Train Loss: 7.6931 | Train Loss Unweighted: 2.7466 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4372 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1365 | Regression_eta: 0.0115 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8739 | Classification_binary: 2.4372 | Regression_cos_phi: 0.3602 | Regression_energy: 1.4959 | Regression_eta: 0.5666 | Regression_pt: 1.5747 | Regression_sin_phi: 0.3846 | LR: 1.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7740s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1225 | max_loss: 0.0034 | max_backward: 0.3931 | max_optimizer_step: 0.1389 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5827.7 | uss_MB: 5069.1 | child_rss_MB: 4024.2 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:08:10,273] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:51,081] INFO : Step 39900/50000 rank0 | Train Loss: 7.4576 | Train Loss Unweighted: 2.5526 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.2656 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1233 | Regression_eta: 0.0139 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8015 | Classification_binary: 2.2656 | Regression_cos_phi: 0.4096 | Regression_energy: 1.3514 | Regression_eta: 0.6832 | Regression_pt: 1.4196 | Regression_sin_phi: 0.5267 | LR: 1.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5615s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0179 | max_backward: 0.2036 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5828.7 | uss_MB: 5067.0 | child_rss_MB: 3996.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:08:51,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:09:32,447] INFO : Step 40000/50000 rank0 | Train Loss: 7.2145 | Train Loss Unweighted: 2.6885 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3908 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1304 | Regression_eta: 0.0082 | Regression_pt: 0.1331 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7733 | Classification_binary: 2.3908 | Regression_cos_phi: 0.3124 | Regression_energy: 1.4286 | Regression_eta: 0.4011 | Regression_pt: 1.5956 | Regression_sin_phi: 0.3127 | LR: 1.86e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5802s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0120 | max_backward: 0.2033 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5827.7 | uss_MB: 5067.0 | child_rss_MB: 3993.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:09:32,663] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:09:32,683] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 16:09:32,858] INFO : Running validation on rank0 +rank0 [2026-10-28 16:09:32,858] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:09:32,859] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:09:34,227] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 16:09:34,374] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:09:34,508] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:09:34,583] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:09:34,652] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:09:34,720] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:09:34,790] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:09:34,861] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:09:34,973] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:09:35,106] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:09:35,237] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:09:35,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:09:35,379] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:09:35,448] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:09:35,518] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:09:35,589] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:09:35,686] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:09:35,692] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:09:35,695] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:09:35,698] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:09:35,699] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:09:35,702] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:09:35,704] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:09:35,705] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:09:35,708] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:09:35,709] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:09:35,712] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:09:35,714] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:09:35,716] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:09:35,718] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:09:35,721] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:09:35,722] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:09:43,408] INFO : VALIDATION | Step=40000/50000 | Train Loss=7.2145 | Valid Loss=7.4020 | Train Loss Unweighted=2.6885 | Valid Loss Unweighted=2.6488 | Stale=0 | ETA=74.3m +rank0 [2026-10-28 16:09:43,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:43,518] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:09:43,521] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:09:43,697] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:45,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:46,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:47,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:47,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:47,854] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:47,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:47,948] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:09:47,960] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:48,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:50,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:50,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:51,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:52,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:52,273] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:52,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:52,354] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:09:52,357] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:09:52,454] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:54,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:55,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:55,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:56,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:56,844] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:56,852] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:09:57,117] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:09:57,537] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:58,978] INFO : Plotted jets +rank0 [2026-10-28 16:10:00,643] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:00,676] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:10:00,912] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:10:01,303] INFO : Plotted number of elements +rank0 [2026-10-28 16:10:02,704] INFO : Plotted jets +rank0 [2026-10-28 16:10:03,772] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:03,806] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 16:10:04,017] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_40000/cld_edm_qq_pf +rank0 [2026-10-28 16:10:04,378] INFO : Plotted number of elements +rank0 [2026-10-28 16:10:05,764] INFO : Plotted jets +rank0 [2026-10-28 16:10:06,836] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:48,537] INFO : Step 40100/50000 rank0 | Train Loss: 6.6656 | Train Loss Unweighted: 2.6344 | Train Loss Components: Classification: 0.0146 | Classification_binary: 2.3900 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1054 | Regression_eta: 0.0090 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6755 | Classification_binary: 2.3900 | Regression_cos_phi: 0.3995 | Regression_energy: 1.1550 | Regression_eta: 0.4431 | Regression_pt: 1.2487 | Regression_sin_phi: 0.3538 | LR: 1.84e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.7885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.3079 | max_optimizer_step: 0.1472 | max_tensorboard_logging: 0.3770 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6040.8 | uss_MB: 5279.4 | child_rss_MB: 3995.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:10:48,748] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 47.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:11:28,582] INFO : Step 40200/50000 rank0 | Train Loss: 7.2354 | Train Loss Unweighted: 2.7217 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4451 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1209 | Regression_eta: 0.0082 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8309 | Classification_binary: 2.4451 | Regression_cos_phi: 0.3822 | Regression_energy: 1.3244 | Regression_eta: 0.4045 | Regression_pt: 1.4045 | Regression_sin_phi: 0.4437 | LR: 1.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6334s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0541 | max_backward: 0.2029 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2991 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6040.8 | uss_MB: 5279.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:11:28,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:09,756] INFO : Step 40300/50000 rank0 | Train Loss: 6.7473 | Train Loss Unweighted: 2.6951 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.4362 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1139 | Regression_eta: 0.0056 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7448 | Classification_binary: 2.4362 | Regression_cos_phi: 0.3621 | Regression_energy: 1.2483 | Regression_eta: 0.2767 | Regression_pt: 1.3557 | Regression_sin_phi: 0.3234 | LR: 1.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0816 | max_forward: 0.0778 | max_loss: 0.0037 | max_backward: 0.3094 | max_optimizer_step: 0.1459 | max_tensorboard_logging: 0.3820 | max_tensorboard_flush: 0.0054 | Memory: rss_MB: 6040.8 | uss_MB: 5279.3 | child_rss_MB: 3989.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:12:09,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:51,090] INFO : Step 40400/50000 rank0 | Train Loss: 6.6618 | Train Loss Unweighted: 2.4133 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.1645 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1071 | Regression_eta: 0.0105 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7980 | Classification_binary: 2.1645 | Regression_cos_phi: 0.4285 | Regression_energy: 1.1740 | Regression_eta: 0.5147 | Regression_pt: 1.2268 | Regression_sin_phi: 0.3552 | LR: 1.79e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8145s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3042 | max_optimizer_step: 0.1480 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6040.8 | uss_MB: 5279.5 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:12:51,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:13:33,322] INFO : Step 40500/50000 rank0 | Train Loss: 7.8518 | Train Loss Unweighted: 2.6107 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2870 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1425 | Regression_eta: 0.0146 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8066 | Classification_binary: 2.2870 | Regression_cos_phi: 0.4167 | Regression_energy: 1.5615 | Regression_eta: 0.7201 | Regression_pt: 1.6369 | Regression_sin_phi: 0.4230 | LR: 1.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7557s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0735 | max_backward: 0.2869 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5899.8 | uss_MB: 5141.4 | child_rss_MB: 3995.8 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:13:33,541] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:15,131] INFO : Step 40600/50000 rank0 | Train Loss: 7.3828 | Train Loss Unweighted: 2.5528 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2621 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1282 | Regression_eta: 0.0134 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7552 | Classification_binary: 2.2621 | Regression_cos_phi: 0.4234 | Regression_energy: 1.4049 | Regression_eta: 0.6572 | Regression_pt: 1.4376 | Regression_sin_phi: 0.4425 | LR: 1.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9108s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1061 | max_loss: 0.0035 | max_backward: 0.3917 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3886 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5899.8 | uss_MB: 5141.5 | child_rss_MB: 3992.3 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:14:15,344] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:55,982] INFO : Step 40700/50000 rank0 | Train Loss: 7.1523 | Train Loss Unweighted: 2.6378 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3613 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1197 | Regression_eta: 0.0082 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7832 | Classification_binary: 2.3613 | Regression_cos_phi: 0.5030 | Regression_energy: 1.3120 | Regression_eta: 0.4035 | Regression_pt: 1.4240 | Regression_sin_phi: 0.3654 | LR: 1.75e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6205s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0657 | max_backward: 0.2030 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5900.8 | uss_MB: 5139.8 | child_rss_MB: 3990.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:14:56,196] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:15:36,607] INFO : Step 40800/50000 rank0 | Train Loss: 6.8422 | Train Loss Unweighted: 2.5608 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3048 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1104 | Regression_eta: 0.0079 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7985 | Classification_binary: 2.3048 | Regression_cos_phi: 0.4799 | Regression_energy: 1.2098 | Regression_eta: 0.3896 | Regression_pt: 1.2931 | Regression_sin_phi: 0.3665 | LR: 1.73e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5849s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0307 | max_backward: 0.2030 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5900.8 | uss_MB: 5139.8 | child_rss_MB: 3991.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:15:36,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:16:19,105] INFO : Step 40900/50000 rank0 | Train Loss: 8.0948 | Train Loss Unweighted: 2.6801 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3827 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1224 | Regression_eta: 0.0231 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8487 | Classification_binary: 2.3827 | Regression_cos_phi: 0.5323 | Regression_energy: 1.3407 | Regression_eta: 1.1362 | Regression_pt: 1.4324 | Regression_sin_phi: 0.4219 | LR: 1.72e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.8268s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2109 | max_loss: 0.0035 | max_backward: 0.2872 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.2885 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5900.8 | uss_MB: 5141.9 | child_rss_MB: 3993.1 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:16:19,319] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:17:00,475] INFO : Step 41000/50000 rank0 | Train Loss: 7.8388 | Train Loss Unweighted: 2.7823 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4684 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1365 | Regression_eta: 0.0102 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9595 | Classification_binary: 2.4684 | Regression_cos_phi: 0.3974 | Regression_energy: 1.4955 | Regression_eta: 0.5013 | Regression_pt: 1.6131 | Regression_sin_phi: 0.4036 | LR: 1.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7214s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1646 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 5900.8 | uss_MB: 5139.8 | child_rss_MB: 3991.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:17:00,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:17:00,708] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 16:17:00,884] INFO : Running validation on rank0 +rank0 [2026-10-28 16:17:00,884] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:17:00,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:17:02,228] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 16:17:02,376] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:17:02,509] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:17:02,583] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:17:02,652] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:17:02,722] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:17:02,791] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:17:02,861] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:17:02,974] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:17:03,105] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:17:03,237] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:17:03,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:17:03,379] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:17:03,449] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:17:03,519] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:17:03,589] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:17:03,688] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:17:03,695] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:17:03,698] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:17:03,700] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:17:03,701] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:17:03,704] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:17:03,705] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:17:03,708] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:17:03,709] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:17:03,712] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:17:03,713] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:17:03,715] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:17:03,718] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:17:03,719] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:17:03,722] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:17:03,724] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:17:11,452] INFO : VALIDATION | Step=41000/50000 | Train Loss=7.8388 | Valid Loss=7.3869 | Train Loss Unweighted=2.7823 | Valid Loss Unweighted=2.6396 | Stale=0 | ETA=66.9m +rank0 [2026-10-28 16:17:11,462] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:11,548] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:17:11,551] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:17:11,720] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:17:13,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:17:14,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:17:15,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:17:15,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:17:16,113] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:17:16,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:16,180] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:17:16,183] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:17:16,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:17:18,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:17:19,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:17:19,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:17:20,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:17:20,788] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:17:20,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:20,851] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:17:20,854] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:17:20,970] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:17:22,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:17:23,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:17:24,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:17:24,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:17:24,966] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:17:24,974] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:17:25,237] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:17:25,643] INFO : Plotted number of elements +rank0 [2026-10-28 16:17:27,081] INFO : Plotted jets +rank0 [2026-10-28 16:17:28,194] INFO : Plotted jet ratio +rank0 [2026-10-28 16:17:28,228] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:17:28,456] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:17:28,850] INFO : Plotted number of elements +rank0 [2026-10-28 16:17:30,813] INFO : Plotted jets +rank0 [2026-10-28 16:17:31,935] INFO : Plotted jet ratio +rank0 [2026-10-28 16:17:31,968] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 16:17:32,175] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_41000/cld_edm_qq_pf +rank0 [2026-10-28 16:17:32,537] INFO : Plotted number of elements +rank0 [2026-10-28 16:17:33,917] INFO : Plotted jets +rank0 [2026-10-28 16:17:34,990] INFO : Plotted jet ratio +rank0 [2026-10-28 16:18:16,925] INFO : Step 41100/50000 rank0 | Train Loss: 7.3974 | Train Loss Unweighted: 2.5715 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2940 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1187 | Regression_eta: 0.0110 | Regression_pt: 0.1141 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8164 | Classification_binary: 2.2940 | Regression_cos_phi: 0.5815 | Regression_energy: 1.3006 | Regression_eta: 0.5396 | Regression_pt: 1.3683 | Regression_sin_phi: 0.4970 | LR: 1.69e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7516s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1046 | max_loss: 0.0191 | max_backward: 0.2868 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.3128 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6098.1 | uss_MB: 5338.2 | child_rss_MB: 3988.4 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:18:17,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:18:58,069] INFO : Step 41200/50000 rank0 | Train Loss: 8.0287 | Train Loss Unweighted: 2.7188 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3959 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1433 | Regression_eta: 0.0141 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8561 | Classification_binary: 2.3959 | Regression_cos_phi: 0.5258 | Regression_energy: 1.5705 | Regression_eta: 0.6953 | Regression_pt: 1.5997 | Regression_sin_phi: 0.3853 | LR: 1.67e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7461s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.2429 | max_optimizer_step: 0.1460 | max_tensorboard_logging: 0.3902 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6098.1 | uss_MB: 5336.1 | child_rss_MB: 3988.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:18:58,284] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:19:38,325] INFO : Step 41300/50000 rank0 | Train Loss: 7.6282 | Train Loss Unweighted: 2.6651 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3701 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1276 | Regression_eta: 0.0128 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8157 | Classification_binary: 2.3701 | Regression_cos_phi: 0.4844 | Regression_energy: 1.3984 | Regression_eta: 0.6277 | Regression_pt: 1.4745 | Regression_sin_phi: 0.4572 | LR: 1.66e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1830 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.2339 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0128 | Memory: rss_MB: 6098.1 | uss_MB: 5336.1 | child_rss_MB: 3985.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:19:38,540] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:20:20,353] INFO : Step 41400/50000 rank0 | Train Loss: 7.1106 | Train Loss Unweighted: 2.6044 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3307 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1186 | Regression_eta: 0.0105 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7733 | Classification_binary: 2.3307 | Regression_cos_phi: 0.4798 | Regression_energy: 1.2996 | Regression_eta: 0.5146 | Regression_pt: 1.3928 | Regression_sin_phi: 0.3199 | LR: 1.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7403s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1138 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.2970 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 6098.1 | uss_MB: 5336.1 | child_rss_MB: 3982.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:20:20,564] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:21:01,388] INFO : Step 41500/50000 rank0 | Train Loss: 7.2370 | Train Loss Unweighted: 2.6382 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3459 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1293 | Regression_eta: 0.0088 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7981 | Classification_binary: 2.3459 | Regression_cos_phi: 0.3713 | Regression_energy: 1.4165 | Regression_eta: 0.4339 | Regression_pt: 1.5109 | Regression_sin_phi: 0.3604 | LR: 1.63e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7995s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0780 | max_loss: 0.0037 | max_backward: 0.3072 | max_optimizer_step: 0.1453 | max_tensorboard_logging: 0.3813 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5972.3 | uss_MB: 5210.8 | child_rss_MB: 3978.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:21:01,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:21:43,039] INFO : Step 41600/50000 rank0 | Train Loss: 7.3561 | Train Loss Unweighted: 2.6124 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.3155 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1311 | Regression_eta: 0.0084 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7653 | Classification_binary: 2.3155 | Regression_cos_phi: 0.4510 | Regression_energy: 1.4366 | Regression_eta: 0.4118 | Regression_pt: 1.5276 | Regression_sin_phi: 0.4484 | LR: 1.61e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8063s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1800 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2449 | max_tensorboard_logging: 0.2855 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5972.3 | uss_MB: 5210.8 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 16:21:43,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:22:24,694] INFO : Step 41700/50000 rank0 | Train Loss: 7.4663 | Train Loss Unweighted: 2.6716 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3820 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1278 | Regression_eta: 0.0108 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8707 | Classification_binary: 2.3820 | Regression_cos_phi: 0.4541 | Regression_energy: 1.4002 | Regression_eta: 0.5309 | Regression_pt: 1.4333 | Regression_sin_phi: 0.3952 | LR: 1.60e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.0543s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.2134 | max_loss: 0.0036 | max_backward: 0.3916 | max_optimizer_step: 0.1454 | max_tensorboard_logging: 0.4073 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 5972.3 | uss_MB: 5212.9 | child_rss_MB: 3983.8 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:22:24,907] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:06,937] INFO : Step 41800/50000 rank0 | Train Loss: 7.0316 | Train Loss Unweighted: 2.6903 | Train Loss Components: Classification: 0.0150 | Classification_binary: 2.4240 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1169 | Regression_eta: 0.0106 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6943 | Classification_binary: 2.4240 | Regression_cos_phi: 0.3705 | Regression_energy: 1.2812 | Regression_eta: 0.5227 | Regression_pt: 1.3473 | Regression_sin_phi: 0.3916 | LR: 1.58e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.8700s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0419 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.3100 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.4098 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 5972.3 | uss_MB: 5211.0 | child_rss_MB: 3988.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 16:23:07,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:47,731] INFO : Step 41900/50000 rank0 | Train Loss: 6.9989 | Train Loss Unweighted: 2.6817 | Train Loss Components: Classification: 0.0150 | Classification_binary: 2.4174 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1164 | Regression_eta: 0.0085 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6971 | Classification_binary: 2.4174 | Regression_cos_phi: 0.4146 | Regression_energy: 1.2749 | Regression_eta: 0.4176 | Regression_pt: 1.3374 | Regression_sin_phi: 0.4399 | LR: 1.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6871s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0890 | max_backward: 0.2037 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.3031 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5972.3 | uss_MB: 5210.9 | child_rss_MB: 3988.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:23:47,945] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:24:28,385] INFO : Step 42000/50000 rank0 | Train Loss: 7.3358 | Train Loss Unweighted: 2.6308 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3459 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1245 | Regression_eta: 0.0112 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7883 | Classification_binary: 2.3459 | Regression_cos_phi: 0.4337 | Regression_energy: 1.3638 | Regression_eta: 0.5498 | Regression_pt: 1.4327 | Regression_sin_phi: 0.4217 | LR: 1.56e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6265s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0284 | max_backward: 0.2033 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3128 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5972.3 | uss_MB: 5211.0 | child_rss_MB: 3987.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:24:28,599] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:24:28,619] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 16:24:28,796] INFO : Running validation on rank0 +rank0 [2026-10-28 16:24:28,796] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:24:28,796] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:24:30,165] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 16:24:30,312] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:24:30,445] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:24:30,518] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:24:30,588] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:24:30,657] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:24:30,727] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:24:30,797] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:24:30,910] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:24:31,043] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:24:31,178] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:24:31,249] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:24:31,320] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:24:31,388] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:24:31,458] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:24:31,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:24:31,629] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:24:31,636] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:24:31,639] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:24:31,642] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:24:31,643] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:24:31,645] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:24:31,646] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:24:31,649] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:24:31,652] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:24:31,653] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:24:31,655] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:24:31,656] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:24:31,659] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:24:31,662] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:24:31,664] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:24:31,666] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:24:39,361] INFO : VALIDATION | Step=42000/50000 | Train Loss=7.3358 | Valid Loss=7.3656 | Train Loss Unweighted=2.6308 | Valid Loss Unweighted=2.6342 | Stale=0 | ETA=59.4m +rank0 [2026-10-28 16:24:39,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:24:39,450] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:24:39,454] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:24:39,634] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:24:41,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:24:42,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:24:42,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:24:43,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:24:43,771] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:24:43,774] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:24:43,847] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:24:43,850] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:24:43,955] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:24:45,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:24:46,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:24:47,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:24:47,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:24:47,970] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:24:47,973] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:24:48,041] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:24:48,044] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:24:48,152] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:24:50,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:24:50,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:24:51,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:24:51,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:24:52,158] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:24:52,165] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:24:52,428] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:24:52,834] INFO : Plotted number of elements +rank0 [2026-10-28 16:24:54,280] INFO : Plotted jets +rank0 [2026-10-28 16:24:56,011] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:56,044] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:24:56,279] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:24:56,671] INFO : Plotted number of elements +rank0 [2026-10-28 16:24:58,064] INFO : Plotted jets +rank0 [2026-10-28 16:24:59,191] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:59,224] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 16:24:59,433] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_42000/cld_edm_qq_pf +rank0 [2026-10-28 16:24:59,799] INFO : Plotted number of elements +rank0 [2026-10-28 16:25:01,176] INFO : Plotted jets +rank0 [2026-10-28 16:25:02,244] INFO : Plotted jet ratio +rank0 [2026-10-28 16:25:44,380] INFO : Step 42100/50000 rank0 | Train Loss: 6.9453 | Train Loss Unweighted: 2.6282 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.3685 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1133 | Regression_eta: 0.0085 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7616 | Classification_binary: 2.3685 | Regression_cos_phi: 0.5189 | Regression_energy: 1.2419 | Regression_eta: 0.4178 | Regression_pt: 1.3042 | Regression_sin_phi: 0.3324 | LR: 1.54e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7066s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1067 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.3063 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6176.7 | uss_MB: 5415.6 | child_rss_MB: 3989.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 16:25:44,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:26:25,793] INFO : Step 42200/50000 rank0 | Train Loss: 7.4732 | Train Loss Unweighted: 2.6061 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3164 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1238 | Regression_eta: 0.0129 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8144 | Classification_binary: 2.3164 | Regression_cos_phi: 0.3915 | Regression_energy: 1.3566 | Regression_eta: 0.6340 | Regression_pt: 1.4630 | Regression_sin_phi: 0.4972 | LR: 1.53e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8470s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1856 | max_loss: 0.0038 | max_backward: 0.2031 | max_optimizer_step: 0.2513 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6176.7 | uss_MB: 5415.6 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:26:26,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:26:35,757] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 16:26:35,757] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 16:27:06,730] INFO : Step 42300/50000 rank0 | Train Loss: 6.6740 | Train Loss Unweighted: 2.5505 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3003 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1093 | Regression_eta: 0.0080 | Regression_pt: 0.1050 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7681 | Classification_binary: 2.3003 | Regression_cos_phi: 0.3820 | Regression_energy: 1.1979 | Regression_eta: 0.3937 | Regression_pt: 1.2592 | Regression_sin_phi: 0.3728 | LR: 1.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9014s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0875 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3066 | max_optimizer_step: 0.1487 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0086 | Memory: rss_MB: 6214.8 | uss_MB: 5453.5 | child_rss_MB: 3994.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:27:06,941] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:27:48,525] INFO : Step 42400/50000 rank0 | Train Loss: 7.4859 | Train Loss Unweighted: 2.6139 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3359 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1193 | Regression_eta: 0.0176 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8328 | Classification_binary: 2.3359 | Regression_cos_phi: 0.4099 | Regression_energy: 1.3071 | Regression_eta: 0.8676 | Regression_pt: 1.3312 | Regression_sin_phi: 0.4015 | LR: 1.50e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8480s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.1049 | max_loss: 0.0929 | max_backward: 0.2872 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6214.8 | uss_MB: 5455.6 | child_rss_MB: 3996.8 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 90.9 +rank0 [2026-10-28 16:27:48,757] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:28:29,597] INFO : Step 42500/50000 rank0 | Train Loss: 7.1315 | Train Loss Unweighted: 2.6342 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3610 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1190 | Regression_eta: 0.0089 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7851 | Classification_binary: 2.3610 | Regression_cos_phi: 0.4659 | Regression_energy: 1.3042 | Regression_eta: 0.4400 | Regression_pt: 1.3852 | Regression_sin_phi: 0.3901 | LR: 1.49e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8788s | Phase Times: max_pre_step_barrier: 0.1056 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0036 | max_backward: 0.3088 | max_optimizer_step: 0.1432 | max_tensorboard_logging: 0.3728 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6041.4 | uss_MB: 5279.8 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:28:29,810] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:10,605] INFO : Step 42600/50000 rank0 | Train Loss: 7.4452 | Train Loss Unweighted: 2.6580 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3860 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1149 | Regression_eta: 0.0147 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7850 | Classification_binary: 2.3860 | Regression_cos_phi: 0.4471 | Regression_energy: 1.2591 | Regression_eta: 0.7216 | Regression_pt: 1.3316 | Regression_sin_phi: 0.5149 | LR: 1.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2104 | max_loss: 0.0039 | max_backward: 0.2871 | max_optimizer_step: 0.1501 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6042.4 | uss_MB: 5282.3 | child_rss_MB: 3992.8 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:29:10,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:51,763] INFO : Step 42700/50000 rank0 | Train Loss: 7.2887 | Train Loss Unweighted: 2.6532 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.3642 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1265 | Regression_eta: 0.0094 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7465 | Classification_binary: 2.3642 | Regression_cos_phi: 0.4577 | Regression_energy: 1.3863 | Regression_eta: 0.4601 | Regression_pt: 1.4932 | Regression_sin_phi: 0.3806 | LR: 1.47e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6965s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0038 | max_backward: 0.2034 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.3874 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6042.4 | uss_MB: 5280.3 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:29:51,974] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:30:32,901] INFO : Step 42800/50000 rank0 | Train Loss: 6.9594 | Train Loss Unweighted: 2.6308 | Train Loss Components: Classification: 0.0155 | Classification_binary: 2.3664 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1170 | Regression_eta: 0.0100 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7169 | Classification_binary: 2.3664 | Regression_cos_phi: 0.3894 | Regression_energy: 1.2815 | Regression_eta: 0.4923 | Regression_pt: 1.3218 | Regression_sin_phi: 0.3911 | LR: 1.45e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7803s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0159 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3086 | max_optimizer_step: 0.1455 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6042.4 | uss_MB: 5280.4 | child_rss_MB: 3995.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:30:33,114] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:31:14,604] INFO : Step 42900/50000 rank0 | Train Loss: 6.7383 | Train Loss Unweighted: 2.6087 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.3536 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1105 | Regression_eta: 0.0079 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7526 | Classification_binary: 2.3536 | Regression_cos_phi: 0.3726 | Regression_energy: 1.2112 | Regression_eta: 0.3893 | Regression_pt: 1.3153 | Regression_sin_phi: 0.3438 | LR: 1.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7888s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0116 | max_forward: 0.0782 | max_loss: 0.0037 | max_backward: 0.3104 | max_optimizer_step: 0.1450 | max_tensorboard_logging: 0.3872 | max_tensorboard_flush: 0.0076 | Memory: rss_MB: 6042.4 | uss_MB: 5280.0 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:31:14,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:31:55,741] INFO : Step 43000/50000 rank0 | Train Loss: 7.3513 | Train Loss Unweighted: 2.6736 | Train Loss Components: Classification: 0.0155 | Classification_binary: 2.3915 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1228 | Regression_eta: 0.0100 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7168 | Classification_binary: 2.3915 | Regression_cos_phi: 0.4890 | Regression_energy: 1.3450 | Regression_eta: 0.4929 | Regression_pt: 1.4295 | Regression_sin_phi: 0.4865 | LR: 1.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1848 | max_loss: 0.0036 | max_backward: 0.2029 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.2856 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6042.4 | uss_MB: 5280.3 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:31:55,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:31:55,984] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 16:31:56,158] INFO : Running validation on rank0 +rank0 [2026-10-28 16:31:56,158] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:31:56,158] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:31:57,520] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 16:31:57,667] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:31:57,801] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:31:57,875] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:31:57,945] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:31:58,014] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:31:58,083] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:31:58,152] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:31:58,265] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:31:58,397] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:31:58,529] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:31:58,601] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:31:58,671] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:31:58,739] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:31:58,808] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:31:58,878] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:31:58,978] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:31:58,984] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:31:58,986] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:31:58,989] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:31:58,990] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:31:58,993] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:31:58,994] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:31:58,997] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:31:58,999] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:31:59,000] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:31:59,003] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:31:59,004] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:31:59,007] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:31:59,009] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:31:59,010] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:31:59,013] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:32:06,726] INFO : VALIDATION | Step=43000/50000 | Train Loss=7.3513 | Valid Loss=7.3589 | Train Loss Unweighted=2.6736 | Valid Loss Unweighted=2.6217 | Stale=0 | ETA=52.0m +rank0 [2026-10-28 16:32:06,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:32:06,860] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:32:06,863] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:32:07,071] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:32:09,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:32:09,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:32:10,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:32:11,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:32:11,331] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:32:11,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:32:11,431] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:32:11,440] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:32:11,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:32:13,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:32:14,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:32:15,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:32:15,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:32:16,021] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:32:16,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:32:16,172] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:32:16,174] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:32:16,363] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:32:18,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:32:18,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:32:19,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:32:20,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:32:20,353] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:32:20,361] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:32:20,622] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:32:21,028] INFO : Plotted number of elements +rank0 [2026-10-28 16:32:22,467] INFO : Plotted jets +rank0 [2026-10-28 16:32:23,562] INFO : Plotted jet ratio +rank0 [2026-10-28 16:32:23,594] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:32:23,822] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:32:24,211] INFO : Plotted number of elements +rank0 [2026-10-28 16:32:26,228] INFO : Plotted jets +rank0 [2026-10-28 16:32:27,349] INFO : Plotted jet ratio +rank0 [2026-10-28 16:32:27,382] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 16:32:27,590] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_43000/cld_edm_qq_pf +rank0 [2026-10-28 16:32:27,952] INFO : Plotted number of elements +rank0 [2026-10-28 16:32:29,331] INFO : Plotted jets +rank0 [2026-10-28 16:32:30,396] INFO : Plotted jet ratio +rank0 [2026-10-28 16:33:12,169] INFO : Step 43100/50000 rank0 | Train Loss: 7.0543 | Train Loss Unweighted: 2.5818 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.3174 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1143 | Regression_eta: 0.0117 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7313 | Classification_binary: 2.3174 | Regression_cos_phi: 0.4519 | Regression_energy: 1.2528 | Regression_eta: 0.5733 | Regression_pt: 1.3158 | Regression_sin_phi: 0.4119 | LR: 1.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6369s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0634 | max_backward: 0.2031 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6243.8 | uss_MB: 5482.6 | child_rss_MB: 3993.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:33:12,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:33:53,463] INFO : Step 43200/50000 rank0 | Train Loss: 7.1700 | Train Loss Unweighted: 2.5837 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3139 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1170 | Regression_eta: 0.0117 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7696 | Classification_binary: 2.3139 | Regression_cos_phi: 0.4726 | Regression_energy: 1.2825 | Regression_eta: 0.5754 | Regression_pt: 1.3312 | Regression_sin_phi: 0.4248 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9081s | Phase Times: max_pre_step_barrier: 0.1050 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3110 | max_optimizer_step: 0.1498 | max_tensorboard_logging: 0.3779 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6243.8 | uss_MB: 5482.6 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:33:53,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:34:35,081] INFO : Step 43300/50000 rank0 | Train Loss: 7.2155 | Train Loss Unweighted: 2.5591 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.2859 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1177 | Regression_eta: 0.0118 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7840 | Classification_binary: 2.2859 | Regression_cos_phi: 0.4922 | Regression_energy: 1.2893 | Regression_eta: 0.5786 | Regression_pt: 1.3566 | Regression_sin_phi: 0.4289 | LR: 1.39e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8022s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2097 | max_loss: 0.0037 | max_backward: 0.2869 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6243.8 | uss_MB: 5484.7 | child_rss_MB: 3989.3 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 16:34:35,294] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:16,482] INFO : Step 43400/50000 rank0 | Train Loss: 7.4231 | Train Loss Unweighted: 2.6635 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3845 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1221 | Regression_eta: 0.0113 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8227 | Classification_binary: 2.3845 | Regression_cos_phi: 0.5108 | Regression_energy: 1.3383 | Regression_eta: 0.5554 | Regression_pt: 1.3610 | Regression_sin_phi: 0.4504 | LR: 1.38e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8321s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2100 | max_loss: 0.0037 | max_backward: 0.2868 | max_optimizer_step: 0.1522 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6243.8 | uss_MB: 5484.8 | child_rss_MB: 3994.3 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 88.9 +rank0 [2026-10-28 16:35:16,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:57,600] INFO : Step 43500/50000 rank0 | Train Loss: 7.8412 | Train Loss Unweighted: 2.6222 | Train Loss Components: Classification: 0.0141 | Classification_binary: 2.3187 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1333 | Regression_eta: 0.0215 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6560 | Classification_binary: 2.3187 | Regression_cos_phi: 0.4751 | Regression_energy: 1.4611 | Regression_eta: 1.0566 | Regression_pt: 1.4527 | Regression_sin_phi: 0.4210 | LR: 1.37e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8675s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0018 | max_forward: 0.1048 | max_loss: 0.0036 | max_backward: 0.3941 | max_optimizer_step: 0.1462 | max_tensorboard_logging: 0.3431 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6121.1 | uss_MB: 5363.6 | child_rss_MB: 3994.6 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 16:35:57,814] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:36:39,508] INFO : Step 43600/50000 rank0 | Train Loss: 8.0592 | Train Loss Unweighted: 2.7722 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.4514 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1422 | Regression_eta: 0.0170 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7677 | Classification_binary: 2.4514 | Regression_cos_phi: 0.4877 | Regression_energy: 1.5578 | Regression_eta: 0.8374 | Regression_pt: 1.5847 | Regression_sin_phi: 0.3725 | LR: 1.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9042s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0969 | max_forward: 0.0774 | max_loss: 0.0036 | max_backward: 0.3084 | max_optimizer_step: 0.1482 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6121.1 | uss_MB: 5362.3 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:36:39,721] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:20,319] INFO : Step 43700/50000 rank0 | Train Loss: 7.3123 | Train Loss Unweighted: 2.6771 | Train Loss Components: Classification: 0.0162 | Classification_binary: 2.3853 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1302 | Regression_eta: 0.0092 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7511 | Classification_binary: 2.3853 | Regression_cos_phi: 0.4061 | Regression_energy: 1.4261 | Regression_eta: 0.4537 | Regression_pt: 1.4886 | Regression_sin_phi: 0.4015 | LR: 1.35e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7694s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2862 | max_optimizer_step: 0.1455 | max_tensorboard_logging: 0.3885 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6121.1 | uss_MB: 5362.5 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:37:20,533] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:38:02,240] INFO : Step 43800/50000 rank0 | Train Loss: 7.3878 | Train Loss Unweighted: 2.6173 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3321 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1250 | Regression_eta: 0.0118 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8046 | Classification_binary: 2.3321 | Regression_cos_phi: 0.4696 | Regression_energy: 1.3700 | Regression_eta: 0.5814 | Regression_pt: 1.4118 | Regression_sin_phi: 0.4185 | LR: 1.34e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7305s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1666 | max_loss: 0.0035 | max_backward: 0.2038 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.2871 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6121.1 | uss_MB: 5362.5 | child_rss_MB: 3992.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:38:02,453] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:38:43,291] INFO : Step 43900/50000 rank0 | Train Loss: 7.7212 | Train Loss Unweighted: 2.6081 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3048 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1333 | Regression_eta: 0.0147 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7781 | Classification_binary: 2.3048 | Regression_cos_phi: 0.5628 | Regression_energy: 1.4601 | Regression_eta: 0.7234 | Regression_pt: 1.4897 | Regression_sin_phi: 0.4024 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7715s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.2997 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6121.1 | uss_MB: 5362.6 | child_rss_MB: 3991.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:38:43,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:39:24,914] INFO : Step 44000/50000 rank0 | Train Loss: 7.9835 | Train Loss Unweighted: 2.6233 | Train Loss Components: Classification: 0.0149 | Classification_binary: 2.3119 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1309 | Regression_eta: 0.0240 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.6935 | Classification_binary: 2.3119 | Regression_cos_phi: 0.3953 | Regression_energy: 1.4346 | Regression_eta: 1.1799 | Regression_pt: 1.5506 | Regression_sin_phi: 0.4177 | LR: 1.32e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7718s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1854 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2233 | max_tensorboard_logging: 0.2883 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6121.1 | uss_MB: 5362.6 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 16:39:25,131] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:39:25,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 16:39:25,327] INFO : Running validation on rank0 +rank0 [2026-10-28 16:39:25,327] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:39:25,327] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:39:26,668] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 16:39:26,815] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:39:26,948] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:39:27,023] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:39:27,093] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:39:27,179] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:39:27,251] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:39:27,322] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:39:27,451] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:39:27,584] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:39:27,716] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:39:27,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:39:27,860] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:39:27,929] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:39:27,998] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:39:28,069] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:39:28,169] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:39:28,177] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:39:28,179] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:39:28,181] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:39:28,183] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:39:28,186] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:39:28,187] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:39:28,190] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:39:28,192] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:39:28,193] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:39:28,196] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:39:28,197] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:39:28,200] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:39:28,202] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:39:28,205] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:39:28,206] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:39:35,837] INFO : VALIDATION | Step=44000/50000 | Train Loss=7.9835 | Valid Loss=7.3530 | Train Loss Unweighted=2.6233 | Valid Loss Unweighted=2.6168 | Stale=0 | ETA=44.6m +rank0 [2026-10-28 16:39:35,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:39:35,905] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:39:35,908] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:39:36,026] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:39:38,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:39:38,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:39:39,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:39:40,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:39:40,449] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:39:40,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:39:40,530] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:39:40,533] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:39:40,633] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:39:42,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:39:43,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:39:43,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:39:44,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:39:44,701] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:39:44,704] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:39:44,760] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:39:44,763] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:39:44,887] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:39:46,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:39:47,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:39:48,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:39:48,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:39:48,871] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:39:48,880] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:39:49,136] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:39:49,544] INFO : Plotted number of elements +rank0 [2026-10-28 16:39:50,986] INFO : Plotted jets +rank0 [2026-10-28 16:39:52,079] INFO : Plotted jet ratio +rank0 [2026-10-28 16:39:52,112] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:39:52,338] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:39:52,730] INFO : Plotted number of elements +rank0 [2026-10-28 16:39:54,719] INFO : Plotted jets +rank0 [2026-10-28 16:39:55,849] INFO : Plotted jet ratio +rank0 [2026-10-28 16:39:55,882] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 16:39:56,088] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_44000/cld_edm_qq_pf +rank0 [2026-10-28 16:39:56,449] INFO : Plotted number of elements +rank0 [2026-10-28 16:39:57,831] INFO : Plotted jets +rank0 [2026-10-28 16:39:58,907] INFO : Plotted jet ratio +rank0 [2026-10-28 16:40:40,606] INFO : Step 44100/50000 rank0 | Train Loss: 7.8008 | Train Loss Unweighted: 2.7684 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.4718 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1297 | Regression_eta: 0.0175 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8000 | Classification_binary: 2.4718 | Regression_cos_phi: 0.4357 | Regression_energy: 1.4214 | Regression_eta: 0.8624 | Regression_pt: 1.4397 | Regression_sin_phi: 0.3698 | LR: 1.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7865s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.2811 | max_optimizer_step: 0.1491 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:40:40,820] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:41:22,599] INFO : Step 44200/50000 rank0 | Train Loss: 7.8458 | Train Loss Unweighted: 2.6598 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3615 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1285 | Regression_eta: 0.0210 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8186 | Classification_binary: 2.3615 | Regression_cos_phi: 0.4403 | Regression_energy: 1.4079 | Regression_eta: 1.0323 | Regression_pt: 1.4306 | Regression_sin_phi: 0.3545 | LR: 1.30e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.7807s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.2768 | max_optimizer_step: 0.1503 | max_tensorboard_logging: 0.3787 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:41:22,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:42:03,794] INFO : Step 44300/50000 rank0 | Train Loss: 6.8814 | Train Loss Unweighted: 2.5129 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2563 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1104 | Regression_eta: 0.0108 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7753 | Classification_binary: 2.2563 | Regression_cos_phi: 0.3963 | Regression_energy: 1.2101 | Regression_eta: 0.5285 | Regression_pt: 1.2718 | Regression_sin_phi: 0.4431 | LR: 1.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8954s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0708 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3087 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 0.3916 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:42:04,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:42:45,090] INFO : Step 44400/50000 rank0 | Train Loss: 7.2231 | Train Loss Unweighted: 2.5734 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2772 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1309 | Regression_eta: 0.0077 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7925 | Classification_binary: 2.2772 | Regression_cos_phi: 0.4492 | Regression_energy: 1.4347 | Regression_eta: 0.3788 | Regression_pt: 1.5420 | Regression_sin_phi: 0.3487 | LR: 1.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8540s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0446 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.3107 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.3819 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6338.1 | uss_MB: 5578.6 | child_rss_MB: 3995.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:42:45,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:43:26,178] INFO : Step 44500/50000 rank0 | Train Loss: 7.4555 | Train Loss Unweighted: 2.6195 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3398 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1207 | Regression_eta: 0.0141 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8442 | Classification_binary: 2.3398 | Regression_cos_phi: 0.4402 | Regression_energy: 1.3223 | Regression_eta: 0.6925 | Regression_pt: 1.3578 | Regression_sin_phi: 0.4586 | LR: 1.27e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6780s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0801 | max_loss: 0.0034 | max_backward: 0.2034 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.3070 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6197.9 | uss_MB: 5437.1 | child_rss_MB: 3991.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:43:26,393] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:44:06,829] INFO : Step 44600/50000 rank0 | Train Loss: 7.8700 | Train Loss Unweighted: 2.5389 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2161 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1377 | Regression_eta: 0.0185 | Regression_pt: 0.1374 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8141 | Classification_binary: 2.2161 | Regression_cos_phi: 0.3750 | Regression_energy: 1.5089 | Regression_eta: 0.9113 | Regression_pt: 1.6474 | Regression_sin_phi: 0.3973 | LR: 1.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8312s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1833 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.3181 | max_tensorboard_flush: 0.0157 | Memory: rss_MB: 6199.9 | uss_MB: 5438.6 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:44:07,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:44:48,611] INFO : Step 44700/50000 rank0 | Train Loss: 7.3791 | Train Loss Unweighted: 2.5887 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3021 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1238 | Regression_eta: 0.0128 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7816 | Classification_binary: 2.3021 | Regression_cos_phi: 0.4705 | Regression_energy: 1.3565 | Regression_eta: 0.6280 | Regression_pt: 1.4413 | Regression_sin_phi: 0.3991 | LR: 1.25e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7067s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0769 | max_backward: 0.2037 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.3072 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6199.9 | uss_MB: 5438.8 | child_rss_MB: 3998.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:44:48,826] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:45:29,934] INFO : Step 44800/50000 rank0 | Train Loss: 7.1036 | Train Loss Unweighted: 2.5417 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.2537 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1274 | Regression_eta: 0.0094 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7683 | Classification_binary: 2.2537 | Regression_cos_phi: 0.3839 | Regression_energy: 1.3962 | Regression_eta: 0.4614 | Regression_pt: 1.4813 | Regression_sin_phi: 0.3587 | LR: 1.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8725s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0018 | max_forward: 0.1041 | max_loss: 0.0034 | max_backward: 0.3932 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.3475 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 6199.9 | uss_MB: 5441.3 | child_rss_MB: 3998.0 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:45:30,147] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:46:11,254] INFO : Step 44900/50000 rank0 | Train Loss: 7.3895 | Train Loss Unweighted: 2.6300 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3597 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1152 | Regression_eta: 0.0115 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7903 | Classification_binary: 2.3597 | Regression_cos_phi: 0.6397 | Regression_energy: 1.2618 | Regression_eta: 0.5667 | Regression_pt: 1.3237 | Regression_sin_phi: 0.4475 | LR: 1.23e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6999s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1334 | max_loss: 0.0034 | max_backward: 0.2035 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6199.9 | uss_MB: 5439.3 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:46:11,470] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:46:52,451] INFO : Step 45000/50000 rank0 | Train Loss: 7.2287 | Train Loss Unweighted: 2.7190 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.4407 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1209 | Regression_eta: 0.0098 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7559 | Classification_binary: 2.4407 | Regression_cos_phi: 0.4100 | Regression_energy: 1.3244 | Regression_eta: 0.4811 | Regression_pt: 1.4315 | Regression_sin_phi: 0.3850 | LR: 1.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6841s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0741 | max_backward: 0.2037 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6199.9 | uss_MB: 5439.4 | child_rss_MB: 3996.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:46:52,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:46:52,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 16:46:52,862] INFO : Running validation on rank0 +rank0 [2026-10-28 16:46:52,862] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:46:52,862] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:46:54,195] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 16:46:54,342] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:46:54,473] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:46:54,547] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:46:54,617] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:46:54,686] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:46:54,756] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:46:54,826] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:46:54,940] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:46:55,071] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:46:55,203] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:46:55,275] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:46:55,346] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:46:55,414] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:46:55,483] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:46:55,553] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:46:55,650] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:46:55,656] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:46:55,659] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:46:55,662] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:46:55,663] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:46:55,673] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:46:55,680] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:46:55,683] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:46:55,686] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:46:55,687] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:46:55,689] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:46:55,692] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:46:55,693] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:46:55,696] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:46:55,697] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:46:55,699] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:47:03,450] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.2287 | Valid Loss=7.3374 | Train Loss Unweighted=2.7190 | Valid Loss Unweighted=2.6159 | Stale=0 | ETA=37.2m +rank0 [2026-10-28 16:47:03,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:47:03,517] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:47:03,520] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:47:03,702] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:47:05,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:47:06,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:47:07,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:47:07,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:47:08,022] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:47:08,025] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:47:08,081] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:47:08,084] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:47:08,199] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:47:10,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:47:10,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:47:11,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:47:12,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:47:12,351] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:47:12,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:47:12,410] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:47:12,413] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:47:12,531] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:47:14,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:47:15,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:47:15,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:47:16,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:47:16,431] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:47:16,439] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:47:16,698] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:47:17,111] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:18,562] INFO : Plotted jets +rank0 [2026-10-28 16:47:19,671] INFO : Plotted jet ratio +rank0 [2026-10-28 16:47:19,705] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:47:19,933] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:47:20,325] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:21,716] INFO : Plotted jets +rank0 [2026-10-28 16:47:23,396] INFO : Plotted jet ratio +rank0 [2026-10-28 16:47:23,429] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 16:47:23,637] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_45000/cld_edm_qq_pf +rank0 [2026-10-28 16:47:24,002] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:25,378] INFO : Plotted jets +rank0 [2026-10-28 16:47:26,440] INFO : Plotted jet ratio +rank0 [2026-10-28 16:48:07,596] INFO : Step 45100/50000 rank0 | Train Loss: 7.6104 | Train Loss Unweighted: 2.4881 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.1962 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1255 | Regression_eta: 0.0154 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8595 | Classification_binary: 2.1962 | Regression_cos_phi: 0.5865 | Regression_energy: 1.3752 | Regression_eta: 0.7547 | Regression_pt: 1.4078 | Regression_sin_phi: 0.4304 | LR: 1.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7954s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1041 | max_loss: 0.0738 | max_backward: 0.2874 | max_optimizer_step: 0.1491 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6405.8 | uss_MB: 5647.5 | child_rss_MB: 3990.5 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:48:07,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:48:48,388] INFO : Step 45200/50000 rank0 | Train Loss: 7.8298 | Train Loss Unweighted: 2.7218 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.4131 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1338 | Regression_eta: 0.0134 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7794 | Classification_binary: 2.4131 | Regression_cos_phi: 0.4656 | Regression_energy: 1.4664 | Regression_eta: 0.6589 | Regression_pt: 1.5644 | Regression_sin_phi: 0.4819 | LR: 1.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6787s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0884 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.2959 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6405.8 | uss_MB: 5645.4 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:48:48,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:49:30,157] INFO : Step 45300/50000 rank0 | Train Loss: 7.2037 | Train Loss Unweighted: 2.7056 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4312 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1213 | Regression_eta: 0.0075 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8098 | Classification_binary: 2.4312 | Regression_cos_phi: 0.4800 | Regression_energy: 1.3289 | Regression_eta: 0.3693 | Regression_pt: 1.3784 | Regression_sin_phi: 0.4060 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7792s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.2886 | max_optimizer_step: 0.1465 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6405.8 | uss_MB: 5645.5 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:49:30,372] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:50:11,263] INFO : Step 45400/50000 rank0 | Train Loss: 6.7664 | Train Loss Unweighted: 2.5806 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.3229 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1131 | Regression_eta: 0.0084 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7317 | Classification_binary: 2.3229 | Regression_cos_phi: 0.4097 | Regression_energy: 1.2392 | Regression_eta: 0.4134 | Regression_pt: 1.3098 | Regression_sin_phi: 0.3397 | LR: 1.19e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8022s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1554 | max_loss: 0.0037 | max_backward: 0.2030 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.3025 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 6405.8 | uss_MB: 5645.1 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:50:11,479] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:50:52,271] INFO : Step 45500/50000 rank0 | Train Loss: 6.9032 | Train Loss Unweighted: 2.5501 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2815 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1157 | Regression_eta: 0.0082 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7748 | Classification_binary: 2.2815 | Regression_cos_phi: 0.3871 | Regression_energy: 1.2676 | Regression_eta: 0.4013 | Regression_pt: 1.3932 | Regression_sin_phi: 0.3976 | LR: 1.18e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.9213s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0409 | max_forward: 0.0773 | max_loss: 0.0036 | max_backward: 0.3101 | max_optimizer_step: 0.1512 | max_tensorboard_logging: 0.4153 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6271.8 | uss_MB: 5510.5 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:50:52,485] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:51:34,302] INFO : Step 45600/50000 rank0 | Train Loss: 7.5787 | Train Loss Unweighted: 2.5387 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2300 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1354 | Regression_eta: 0.0121 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8265 | Classification_binary: 2.2300 | Regression_cos_phi: 0.4710 | Regression_energy: 1.4833 | Regression_eta: 0.5928 | Regression_pt: 1.5625 | Regression_sin_phi: 0.4127 | LR: 1.17e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0411 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3100 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.3953 | max_tensorboard_flush: 0.0196 | Memory: rss_MB: 6272.8 | uss_MB: 5511.6 | child_rss_MB: 3996.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:51:34,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:15,392] INFO : Step 45700/50000 rank0 | Train Loss: 7.4985 | Train Loss Unweighted: 2.6312 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3257 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1352 | Regression_eta: 0.0098 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7759 | Classification_binary: 2.3257 | Regression_cos_phi: 0.4876 | Regression_energy: 1.4815 | Regression_eta: 0.4837 | Regression_pt: 1.5687 | Regression_sin_phi: 0.3754 | LR: 1.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7997s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1336 | max_loss: 0.0037 | max_backward: 0.2032 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.3454 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6272.8 | uss_MB: 5511.8 | child_rss_MB: 4014.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:52:15,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:56,398] INFO : Step 45800/50000 rank0 | Train Loss: 7.7721 | Train Loss Unweighted: 2.6494 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3672 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1177 | Regression_eta: 0.0143 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8149 | Classification_binary: 2.3672 | Regression_cos_phi: 0.6675 | Regression_energy: 1.2899 | Regression_eta: 0.7037 | Regression_pt: 1.3697 | Regression_sin_phi: 0.5591 | LR: 1.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2111 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0123 | Memory: rss_MB: 6272.8 | uss_MB: 5512.0 | child_rss_MB: 3999.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:52:56,611] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:53:38,341] INFO : Step 45900/50000 rank0 | Train Loss: 7.5350 | Train Loss Unweighted: 2.7039 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4147 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1249 | Regression_eta: 0.0113 | Regression_pt: 0.1216 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8244 | Classification_binary: 2.4147 | Regression_cos_phi: 0.4713 | Regression_energy: 1.3685 | Regression_eta: 0.5560 | Regression_pt: 1.4579 | Regression_sin_phi: 0.4422 | LR: 1.15e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7208s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1491 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.2771 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6272.8 | uss_MB: 5512.4 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:53:38,555] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:54:20,371] INFO : Step 46000/50000 rank0 | Train Loss: 7.7253 | Train Loss Unweighted: 2.6006 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3084 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1257 | Regression_eta: 0.0179 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7771 | Classification_binary: 2.3084 | Regression_cos_phi: 0.5385 | Regression_energy: 1.3777 | Regression_eta: 0.8818 | Regression_pt: 1.4056 | Regression_sin_phi: 0.4362 | LR: 1.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7877s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1858 | max_loss: 0.0037 | max_backward: 0.2031 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6272.8 | uss_MB: 5512.4 | child_rss_MB: 3994.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:54:20,586] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:54:20,606] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 16:54:20,779] INFO : Running validation on rank0 +rank0 [2026-10-28 16:54:20,779] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:54:20,779] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:54:22,102] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 16:54:22,249] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:54:22,382] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:54:22,456] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:54:22,525] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:54:22,595] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:54:22,665] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:54:22,734] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:54:22,847] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:54:22,979] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:54:23,112] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:54:23,184] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:54:23,253] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:54:23,322] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:54:23,393] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:54:23,463] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:54:23,562] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:54:23,568] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:54:23,571] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:54:23,574] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:54:23,577] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:54:23,578] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:54:23,581] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:54:23,582] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:54:23,584] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:54:23,587] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:54:23,590] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:54:23,591] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:54:23,594] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:54:23,597] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:54:23,599] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:54:23,600] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:54:31,310] INFO : VALIDATION | Step=46000/50000 | Train Loss=7.7253 | Valid Loss=7.3077 | Train Loss Unweighted=2.6006 | Valid Loss Unweighted=2.6065 | Stale=0 | ETA=29.7m +rank0 [2026-10-28 16:54:31,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:54:31,401] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:54:31,405] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 16:54:31,509] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:54:33,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:54:34,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:54:34,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:54:35,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:54:35,737] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:54:35,740] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:54:35,845] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:54:35,848] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:54:36,037] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:54:38,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:54:38,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:54:39,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:54:39,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:54:40,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:54:40,099] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:54:40,188] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 16:54:40,208] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 16:54:40,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:54:42,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:54:42,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:54:43,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:54:44,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:54:44,289] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:54:44,296] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:54:44,555] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ttbar_pf +rank0 [2026-10-28 16:54:44,961] INFO : Plotted number of elements +rank0 [2026-10-28 16:54:46,398] INFO : Plotted jets +rank0 [2026-10-28 16:54:47,510] INFO : Plotted jet ratio +rank0 [2026-10-28 16:54:47,544] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:54:47,774] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 16:54:48,163] INFO : Plotted number of elements +rank0 [2026-10-28 16:54:49,549] INFO : Plotted jets +rank0 [2026-10-28 16:54:51,230] INFO : Plotted jet ratio +rank0 [2026-10-28 16:54:51,266] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 16:54:51,473] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_46000/cld_edm_qq_pf +rank0 [2026-10-28 16:54:51,833] INFO : Plotted number of elements +rank0 [2026-10-28 16:54:53,204] INFO : Plotted jets +rank0 [2026-10-28 16:54:54,277] INFO : Plotted jet ratio +rank0 [2026-10-28 16:55:36,066] INFO : Step 46100/50000 rank0 | Train Loss: 7.3680 | Train Loss Unweighted: 2.6031 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3072 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1306 | Regression_eta: 0.0124 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7874 | Classification_binary: 2.3072 | Regression_cos_phi: 0.3754 | Regression_energy: 1.4315 | Regression_eta: 0.6097 | Regression_pt: 1.4982 | Regression_sin_phi: 0.3587 | LR: 1.13e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6795s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0848 | max_backward: 0.2033 | max_optimizer_step: 0.1490 | max_tensorboard_logging: 0.2794 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6469.9 | uss_MB: 5711.7 | child_rss_MB: 3994.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:55:36,280] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:56:17,489] INFO : Step 46200/50000 rank0 | Train Loss: 6.8115 | Train Loss Unweighted: 2.5816 | Train Loss Components: Classification: 0.0155 | Classification_binary: 2.3171 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1155 | Regression_eta: 0.0083 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7183 | Classification_binary: 2.3171 | Regression_cos_phi: 0.3785 | Regression_energy: 1.2654 | Regression_eta: 0.4087 | Regression_pt: 1.3694 | Regression_sin_phi: 0.3541 | LR: 1.13e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6383s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.1532 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6469.9 | uss_MB: 5711.7 | child_rss_MB: 3995.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:56:17,706] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:56:58,855] INFO : Step 46300/50000 rank0 | Train Loss: 7.1715 | Train Loss Unweighted: 2.5812 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.2874 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1299 | Regression_eta: 0.0080 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7368 | Classification_binary: 2.2874 | Regression_cos_phi: 0.3994 | Regression_energy: 1.4235 | Regression_eta: 0.3949 | Regression_pt: 1.5357 | Regression_sin_phi: 0.3937 | LR: 1.12e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7886s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2619 | max_optimizer_step: 0.1519 | max_tensorboard_logging: 0.3854 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6469.9 | uss_MB: 5711.7 | child_rss_MB: 3998.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:56:59,070] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:57:40,759] INFO : Step 46400/50000 rank0 | Train Loss: 7.1005 | Train Loss Unweighted: 2.5478 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2718 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1210 | Regression_eta: 0.0090 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8092 | Classification_binary: 2.2718 | Regression_cos_phi: 0.4850 | Regression_energy: 1.3257 | Regression_eta: 0.4401 | Regression_pt: 1.3876 | Regression_sin_phi: 0.3812 | LR: 1.11e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.9380s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1043 | max_loss: 0.0035 | max_backward: 0.3946 | max_optimizer_step: 0.1513 | max_tensorboard_logging: 0.3798 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6469.9 | uss_MB: 5713.8 | child_rss_MB: 3997.4 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:57:40,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:58:21,496] INFO : Step 46500/50000 rank0 | Train Loss: 7.8905 | Train Loss Unweighted: 2.7863 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4856 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1294 | Regression_eta: 0.0142 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8659 | Classification_binary: 2.4856 | Regression_cos_phi: 0.4960 | Regression_energy: 1.4182 | Regression_eta: 0.6998 | Regression_pt: 1.4926 | Regression_sin_phi: 0.4323 | LR: 1.11e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8673s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0190 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3095 | max_optimizer_step: 0.1545 | max_tensorboard_logging: 0.4070 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6344.7 | uss_MB: 5585.6 | child_rss_MB: 3993.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:58:21,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:02,578] INFO : Step 46600/50000 rank0 | Train Loss: 6.8535 | Train Loss Unweighted: 2.4884 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.2254 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1159 | Regression_eta: 0.0090 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7901 | Classification_binary: 2.2254 | Regression_cos_phi: 0.4145 | Regression_energy: 1.2695 | Regression_eta: 0.4447 | Regression_pt: 1.3053 | Regression_sin_phi: 0.4039 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6817s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1179 | max_loss: 0.0035 | max_backward: 0.2038 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6346.7 | uss_MB: 5587.1 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:59:02,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:44,677] INFO : Step 46700/50000 rank0 | Train Loss: 7.7857 | Train Loss Unweighted: 2.7883 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4895 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1284 | Regression_eta: 0.0120 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8264 | Classification_binary: 2.4895 | Regression_cos_phi: 0.5095 | Regression_energy: 1.4069 | Regression_eta: 0.5892 | Regression_pt: 1.5139 | Regression_sin_phi: 0.4503 | LR: 1.10e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7761s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1829 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2099 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6346.7 | uss_MB: 5587.4 | child_rss_MB: 3993.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:59:44,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:00:25,824] INFO : Step 46800/50000 rank0 | Train Loss: 6.9130 | Train Loss Unweighted: 2.5616 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3120 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1045 | Regression_eta: 0.0120 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7752 | Classification_binary: 2.3120 | Regression_cos_phi: 0.4552 | Regression_energy: 1.1455 | Regression_eta: 0.5908 | Regression_pt: 1.2441 | Regression_sin_phi: 0.3901 | LR: 1.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0828 | max_backward: 0.2034 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6346.7 | uss_MB: 5587.6 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:00:26,038] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:06,507] INFO : Step 46900/50000 rank0 | Train Loss: 7.1149 | Train Loss Unweighted: 2.6090 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3426 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1151 | Regression_eta: 0.0111 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8176 | Classification_binary: 2.3426 | Regression_cos_phi: 0.4367 | Regression_energy: 1.2615 | Regression_eta: 0.5434 | Regression_pt: 1.3211 | Regression_sin_phi: 0.3920 | LR: 1.09e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.9438s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0488 | max_forward: 0.1041 | max_loss: 0.0035 | max_backward: 0.3961 | max_optimizer_step: 0.1459 | max_tensorboard_logging: 0.3713 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6346.7 | uss_MB: 5590.1 | child_rss_MB: 3994.1 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 17:01:06,720] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 49.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:48,275] INFO : Step 47000/50000 rank0 | Train Loss: 6.9266 | Train Loss Unweighted: 2.6062 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3536 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1071 | Regression_eta: 0.0089 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8743 | Classification_binary: 2.3536 | Regression_cos_phi: 0.4315 | Regression_energy: 1.1739 | Regression_eta: 0.4376 | Regression_pt: 1.2649 | Regression_sin_phi: 0.3909 | LR: 1.08e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7060s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1374 | max_loss: 0.0035 | max_backward: 0.2038 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.2852 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 6346.7 | uss_MB: 5588.1 | child_rss_MB: 3998.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:01:48,490] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:48,509] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 17:01:48,686] INFO : Running validation on rank0 +rank0 [2026-10-28 17:01:48,686] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:01:48,686] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:01:50,013] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 17:01:50,158] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:01:50,291] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:01:50,366] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:01:50,435] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:01:50,504] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:01:50,573] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:01:50,642] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:01:50,755] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:01:50,886] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:01:51,019] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:01:51,090] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:01:51,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:01:51,229] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:01:51,299] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:01:51,370] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:01:51,470] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:01:51,477] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:01:51,479] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:01:51,482] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:01:51,483] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:01:51,486] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:01:51,488] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:01:51,491] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:01:51,492] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:01:51,495] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:01:51,497] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:01:51,500] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:01:51,501] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:01:51,503] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:01:51,504] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:01:51,507] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:01:59,072] INFO : VALIDATION | Step=47000/50000 | Train Loss=6.9266 | Valid Loss=7.3067 | Train Loss Unweighted=2.6062 | Valid Loss Unweighted=2.6053 | Stale=0 | ETA=22.3m +rank0 [2026-10-28 17:01:59,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:01:59,138] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:01:59,142] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:01:59,325] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:02:01,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:02:02,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:02:02,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:02:03,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:02:03,623] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:02:03,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:02:03,697] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:02:03,700] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:02:03,809] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:02:05,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:02:06,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:02:07,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:02:07,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:02:07,987] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:02:07,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:02:08,065] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:02:08,068] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:02:08,178] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:02:10,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:02:11,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:02:11,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:02:12,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:02:12,590] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:02:12,598] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:02:12,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:02:13,269] INFO : Plotted number of elements +rank0 [2026-10-28 17:02:14,719] INFO : Plotted jets +rank0 [2026-10-28 17:02:15,821] INFO : Plotted jet ratio +rank0 [2026-10-28 17:02:15,854] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:02:16,082] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:02:16,475] INFO : Plotted number of elements +rank0 [2026-10-28 17:02:17,864] INFO : Plotted jets +rank0 [2026-10-28 17:02:18,990] INFO : Plotted jet ratio +rank0 [2026-10-28 17:02:19,024] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 17:02:19,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_47000/cld_edm_qq_pf +rank0 [2026-10-28 17:02:20,243] INFO : Plotted number of elements +rank0 [2026-10-28 17:02:21,623] INFO : Plotted jets +rank0 [2026-10-28 17:02:22,695] INFO : Plotted jet ratio +rank0 [2026-10-28 17:03:04,724] INFO : Step 47100/50000 rank0 | Train Loss: 7.3951 | Train Loss Unweighted: 2.6919 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.4087 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1224 | Regression_eta: 0.0100 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7692 | Classification_binary: 2.4087 | Regression_cos_phi: 0.5522 | Regression_energy: 1.3409 | Regression_eta: 0.4904 | Regression_pt: 1.4415 | Regression_sin_phi: 0.3922 | LR: 1.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1137 | max_loss: 0.0038 | max_backward: 0.2039 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:03:04,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:03:46,160] INFO : Step 47200/50000 rank0 | Train Loss: 7.3369 | Train Loss Unweighted: 2.5812 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.2925 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1256 | Regression_eta: 0.0125 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7889 | Classification_binary: 2.2925 | Regression_cos_phi: 0.3871 | Regression_energy: 1.3762 | Regression_eta: 0.6162 | Regression_pt: 1.4570 | Regression_sin_phi: 0.4190 | LR: 1.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1530 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.2973 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3995.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:03:46,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:04:28,162] INFO : Step 47300/50000 rank0 | Train Loss: 8.0476 | Train Loss Unweighted: 2.6238 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3205 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1271 | Regression_eta: 0.0201 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8781 | Classification_binary: 2.3205 | Regression_cos_phi: 0.5362 | Regression_energy: 1.3927 | Regression_eta: 0.9862 | Regression_pt: 1.4651 | Regression_sin_phi: 0.4688 | LR: 1.06e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7757s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1850 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.2794 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3995.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:04:28,377] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:05:09,873] INFO : Step 47400/50000 rank0 | Train Loss: 6.8776 | Train Loss Unweighted: 2.4663 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.1906 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1209 | Regression_eta: 0.0080 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7885 | Classification_binary: 2.1906 | Regression_cos_phi: 0.3671 | Regression_energy: 1.3245 | Regression_eta: 0.3957 | Regression_pt: 1.4185 | Regression_sin_phi: 0.3926 | LR: 1.06e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.9385s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1216 | max_loss: 0.0039 | max_backward: 0.3930 | max_optimizer_step: 0.1500 | max_tensorboard_logging: 0.3746 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6556.0 | uss_MB: 5798.5 | child_rss_MB: 3994.9 | allocated_MB: 295.1 | reserved_MB: 3840.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 17:05:10,085] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:05:51,662] INFO : Step 47500/50000 rank0 | Train Loss: 7.5574 | Train Loss Unweighted: 2.6618 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.3583 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1329 | Regression_eta: 0.0090 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7586 | Classification_binary: 2.3583 | Regression_cos_phi: 0.4805 | Regression_energy: 1.4561 | Regression_eta: 0.4407 | Regression_pt: 1.5665 | Regression_sin_phi: 0.4968 | LR: 1.06e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1809 | max_loss: 0.0037 | max_backward: 0.2036 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6556.0 | uss_MB: 5796.4 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:05:51,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:33,719] INFO : Step 47600/50000 rank0 | Train Loss: 7.2782 | Train Loss Unweighted: 2.5998 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3153 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1228 | Regression_eta: 0.0076 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8113 | Classification_binary: 2.3153 | Regression_cos_phi: 0.5821 | Regression_energy: 1.3456 | Regression_eta: 0.3759 | Regression_pt: 1.4660 | Regression_sin_phi: 0.3820 | LR: 1.05e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6573s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0796 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6419.2 | uss_MB: 5659.2 | child_rss_MB: 3996.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:06:33,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:07:14,841] INFO : Step 47700/50000 rank0 | Train Loss: 7.4307 | Train Loss Unweighted: 2.5335 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2494 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1236 | Regression_eta: 0.0165 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7574 | Classification_binary: 2.2494 | Regression_cos_phi: 0.4882 | Regression_energy: 1.3543 | Regression_eta: 0.8115 | Regression_pt: 1.3714 | Regression_sin_phi: 0.3986 | LR: 1.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7976s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.2740 | max_optimizer_step: 0.1516 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6419.2 | uss_MB: 5659.3 | child_rss_MB: 3995.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:07:15,057] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:07:56,649] INFO : Step 47800/50000 rank0 | Train Loss: 7.1418 | Train Loss Unweighted: 2.6018 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3357 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1153 | Regression_eta: 0.0118 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8480 | Classification_binary: 2.3357 | Regression_cos_phi: 0.4073 | Regression_energy: 1.2631 | Regression_eta: 0.5782 | Regression_pt: 1.3025 | Regression_sin_phi: 0.4070 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.2788s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.6627 | max_loss: 0.0037 | max_backward: 0.2038 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.2847 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6420.2 | uss_MB: 5659.9 | child_rss_MB: 3997.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:07:56,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:08:37,742] INFO : Step 47900/50000 rank0 | Train Loss: 7.1295 | Train Loss Unweighted: 2.4979 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.2232 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1181 | Regression_eta: 0.0124 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7613 | Classification_binary: 2.2232 | Regression_cos_phi: 0.4629 | Regression_energy: 1.2942 | Regression_eta: 0.6102 | Regression_pt: 1.3774 | Regression_sin_phi: 0.4002 | LR: 1.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7195s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.2196 | max_optimizer_step: 0.1489 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6420.2 | uss_MB: 5659.9 | child_rss_MB: 3992.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 17:08:37,957] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:09:17,752] INFO : Step 48000/50000 rank0 | Train Loss: 6.8565 | Train Loss Unweighted: 2.5352 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2814 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1084 | Regression_eta: 0.0097 | Regression_pt: 0.1059 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8103 | Classification_binary: 2.2814 | Regression_cos_phi: 0.4439 | Regression_energy: 1.1872 | Regression_eta: 0.4787 | Regression_pt: 1.2697 | Regression_sin_phi: 0.3853 | LR: 1.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6191s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0153 | max_backward: 0.2034 | max_optimizer_step: 0.1517 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6420.2 | uss_MB: 5660.1 | child_rss_MB: 3992.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:09:17,966] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:09:17,985] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 17:09:18,166] INFO : Running validation on rank0 +rank0 [2026-10-28 17:09:18,167] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:09:18,167] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:09:19,557] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 17:09:19,706] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:09:19,837] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:09:19,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:09:19,981] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:09:20,050] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:09:20,120] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:09:20,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:09:20,303] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:09:20,435] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:09:20,567] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:09:20,638] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:09:20,708] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:09:20,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:09:20,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:09:20,916] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:09:21,015] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:09:21,022] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:09:21,024] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:09:21,027] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:09:21,028] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:09:21,031] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:09:21,032] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:09:21,035] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:09:21,037] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:09:21,039] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:09:21,041] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:09:21,042] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:09:21,045] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:09:21,048] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:09:21,050] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:09:21,052] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:09:28,763] INFO : VALIDATION | Step=48000/50000 | Train Loss=6.8565 | Valid Loss=7.2882 | Train Loss Unweighted=2.5352 | Valid Loss Unweighted=2.6060 | Stale=0 | ETA=14.9m +rank0 [2026-10-28 17:09:28,774] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:28,920] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:09:28,924] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:09:29,116] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:09:31,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:09:31,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:09:32,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:09:33,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:09:33,577] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:09:33,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:33,655] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:09:33,658] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:09:33,767] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:09:36,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:09:36,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:09:37,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:09:38,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:09:38,254] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:09:38,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:38,327] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:09:38,330] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:09:38,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:09:40,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:09:41,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:09:41,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:09:42,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:09:42,529] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:09:42,537] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:09:42,798] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:09:43,214] INFO : Plotted number of elements +rank0 [2026-10-28 17:09:44,659] INFO : Plotted jets +rank0 [2026-10-28 17:09:45,764] INFO : Plotted jet ratio +rank0 [2026-10-28 17:09:45,797] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:09:46,031] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:09:46,424] INFO : Plotted number of elements +rank0 [2026-10-28 17:09:47,824] INFO : Plotted jets +rank0 [2026-10-28 17:09:49,563] INFO : Plotted jet ratio +rank0 [2026-10-28 17:09:49,596] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 17:09:49,807] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_48000/cld_edm_qq_pf +rank0 [2026-10-28 17:09:50,171] INFO : Plotted number of elements +rank0 [2026-10-28 17:09:51,556] INFO : Plotted jets +rank0 [2026-10-28 17:09:52,632] INFO : Plotted jet ratio +rank0 [2026-10-28 17:10:34,996] INFO : Step 48100/50000 rank0 | Train Loss: 7.2827 | Train Loss Unweighted: 2.6399 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3723 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1147 | Regression_eta: 0.0127 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7825 | Classification_binary: 2.3723 | Regression_cos_phi: 0.4974 | Regression_energy: 1.2568 | Regression_eta: 0.6220 | Regression_pt: 1.3108 | Regression_sin_phi: 0.4409 | LR: 1.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.0407s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1024 | max_forward: 0.1044 | max_loss: 0.0035 | max_backward: 0.3940 | max_optimizer_step: 0.1521 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6604.5 | uss_MB: 5847.7 | child_rss_MB: 3989.5 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 17:10:35,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:11:15,904] INFO : Step 48200/50000 rank0 | Train Loss: 7.2577 | Train Loss Unweighted: 2.5515 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2676 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1246 | Regression_eta: 0.0102 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7932 | Classification_binary: 2.2676 | Regression_cos_phi: 0.4694 | Regression_energy: 1.3656 | Regression_eta: 0.4992 | Regression_pt: 1.4198 | Regression_sin_phi: 0.4429 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6713s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3995.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:11:16,118] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:11:57,564] INFO : Step 48300/50000 rank0 | Train Loss: 7.2231 | Train Loss Unweighted: 2.5918 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3117 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1211 | Regression_eta: 0.0090 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8550 | Classification_binary: 2.3117 | Regression_cos_phi: 0.4174 | Regression_energy: 1.3272 | Regression_eta: 0.4439 | Regression_pt: 1.4207 | Regression_sin_phi: 0.4472 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8188s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0780 | max_loss: 0.0037 | max_backward: 0.3066 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 0.3720 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3996.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 17:11:57,781] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:12:39,251] INFO : Step 48400/50000 rank0 | Train Loss: 7.8885 | Train Loss Unweighted: 2.6583 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3429 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1387 | Regression_eta: 0.0096 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9110 | Classification_binary: 2.3429 | Regression_cos_phi: 0.6339 | Regression_energy: 1.5200 | Regression_eta: 0.4730 | Regression_pt: 1.5792 | Regression_sin_phi: 0.4284 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7923s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0037 | max_backward: 0.2030 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 0.2855 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3996.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:12:39,464] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:13:20,360] INFO : Step 48500/50000 rank0 | Train Loss: 6.9111 | Train Loss Unweighted: 2.5033 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2449 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1121 | Regression_eta: 0.0123 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7725 | Classification_binary: 2.2449 | Regression_cos_phi: 0.4318 | Regression_energy: 1.2278 | Regression_eta: 0.6040 | Regression_pt: 1.2658 | Regression_sin_phi: 0.3643 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7421s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0779 | max_loss: 0.0037 | max_backward: 0.2029 | max_optimizer_step: 0.1539 | max_tensorboard_logging: 0.3762 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6604.5 | uss_MB: 5845.6 | child_rss_MB: 3994.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:13:20,574] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:01,026] INFO : Step 48600/50000 rank0 | Train Loss: 6.6640 | Train Loss Unweighted: 2.4674 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.2130 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1106 | Regression_eta: 0.0074 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8750 | Classification_binary: 2.2130 | Regression_cos_phi: 0.3803 | Regression_energy: 1.2120 | Regression_eta: 0.3630 | Regression_pt: 1.2793 | Regression_sin_phi: 0.3414 | LR: 1.02e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6422s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0377 | max_backward: 0.2031 | max_optimizer_step: 0.1500 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6493.2 | uss_MB: 5734.8 | child_rss_MB: 3994.0 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:14:01,239] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:41,982] INFO : Step 48700/50000 rank0 | Train Loss: 7.1915 | Train Loss Unweighted: 2.6255 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3568 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1165 | Regression_eta: 0.0101 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8297 | Classification_binary: 2.3568 | Regression_cos_phi: 0.4324 | Regression_energy: 1.2762 | Regression_eta: 0.4954 | Regression_pt: 1.3273 | Regression_sin_phi: 0.4738 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8166s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1849 | max_loss: 0.0037 | max_backward: 0.2033 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6493.2 | uss_MB: 5735.1 | child_rss_MB: 3991.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:14:42,195] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:15:23,121] INFO : Step 48800/50000 rank0 | Train Loss: 6.7459 | Train Loss Unweighted: 2.4737 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.2159 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1113 | Regression_eta: 0.0071 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7456 | Classification_binary: 2.2159 | Regression_cos_phi: 0.4682 | Regression_energy: 1.2199 | Regression_eta: 0.3477 | Regression_pt: 1.3176 | Regression_sin_phi: 0.4311 | LR: 1.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8162s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1043 | max_loss: 0.1048 | max_backward: 0.2876 | max_optimizer_step: 0.1479 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6493.2 | uss_MB: 5737.5 | child_rss_MB: 3996.3 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 17:15:23,335] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:16:04,511] INFO : Step 48900/50000 rank0 | Train Loss: 7.3672 | Train Loss Unweighted: 2.6951 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4193 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1214 | Regression_eta: 0.0110 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8226 | Classification_binary: 2.4193 | Regression_cos_phi: 0.5183 | Regression_energy: 1.3307 | Regression_eta: 0.5413 | Regression_pt: 1.3436 | Regression_sin_phi: 0.3914 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8409s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0284 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3090 | max_optimizer_step: 0.1496 | max_tensorboard_logging: 0.3764 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6493.2 | uss_MB: 5735.5 | child_rss_MB: 3996.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:16:04,723] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:16:45,370] INFO : Step 49000/50000 rank0 | Train Loss: 6.8608 | Train Loss Unweighted: 2.5118 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2559 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1103 | Regression_eta: 0.0096 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7934 | Classification_binary: 2.2559 | Regression_cos_phi: 0.4729 | Regression_energy: 1.2086 | Regression_eta: 0.4737 | Regression_pt: 1.2720 | Regression_sin_phi: 0.3844 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.8192s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0037 | max_backward: 0.3049 | max_optimizer_step: 0.1524 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6493.2 | uss_MB: 5735.6 | child_rss_MB: 3994.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 17:16:45,585] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:16:45,616] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 17:16:45,796] INFO : Running validation on rank0 +rank0 [2026-10-28 17:16:45,797] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:16:45,797] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:16:47,132] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 17:16:47,280] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:16:47,412] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:16:47,487] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:16:47,557] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:16:47,627] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:16:47,697] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:16:47,767] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:16:47,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:16:48,014] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:16:48,147] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:16:48,219] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:16:48,289] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:16:48,358] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:16:48,427] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:16:48,497] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:16:48,598] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:16:48,605] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:16:48,607] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:16:48,610] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:16:48,611] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:16:48,614] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:16:48,617] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:16:48,618] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:16:48,620] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:16:48,623] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:16:48,624] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:16:48,627] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:16:48,630] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:16:48,631] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:16:48,633] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:16:48,636] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:16:56,258] INFO : VALIDATION | Step=49000/50000 | Train Loss=6.8608 | Valid Loss=7.2898 | Train Loss Unweighted=2.5118 | Valid Loss Unweighted=2.6059 | Stale=1 | ETA=7.4m +rank0 [2026-10-28 17:16:56,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:16:56,368] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:16:56,371] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:16:56,623] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:16:58,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:16:59,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:16:59,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:17:00,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:17:00,751] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:17:00,754] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:17:00,808] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:17:00,811] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:17:00,938] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:17:03,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:17:03,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:17:04,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:17:05,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:17:05,350] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:17:05,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:17:05,407] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:17:05,410] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:17:05,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:17:07,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:17:08,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:17:08,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:17:09,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:17:09,584] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:17:09,592] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:17:09,856] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:17:10,267] INFO : Plotted number of elements +rank0 [2026-10-28 17:17:11,714] INFO : Plotted jets +rank0 [2026-10-28 17:17:12,822] INFO : Plotted jet ratio +rank0 [2026-10-28 17:17:12,856] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:17:13,085] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:17:13,480] INFO : Plotted number of elements +rank0 [2026-10-28 17:17:14,876] INFO : Plotted jets +rank0 [2026-10-28 17:17:16,653] INFO : Plotted jet ratio +rank0 [2026-10-28 17:17:16,686] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 17:17:16,892] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_49000/cld_edm_qq_pf +rank0 [2026-10-28 17:17:17,255] INFO : Plotted number of elements +rank0 [2026-10-28 17:17:18,630] INFO : Plotted jets +rank0 [2026-10-28 17:17:19,695] INFO : Plotted jet ratio +rank0 [2026-10-28 17:18:00,771] INFO : Step 49100/50000 rank0 | Train Loss: 7.1387 | Train Loss Unweighted: 2.6347 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.3740 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1119 | Regression_eta: 0.0124 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7431 | Classification_binary: 2.3740 | Regression_cos_phi: 0.4947 | Regression_energy: 1.2264 | Regression_eta: 0.6103 | Regression_pt: 1.2807 | Regression_sin_phi: 0.4094 | LR: 1.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6690s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0167 | max_backward: 0.2038 | max_optimizer_step: 0.1541 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6685.8 | uss_MB: 5927.5 | child_rss_MB: 3990.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:18:00,982] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:18:42,758] INFO : Step 49200/50000 rank0 | Train Loss: 7.0735 | Train Loss Unweighted: 2.5507 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.2835 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1174 | Regression_eta: 0.0112 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7065 | Classification_binary: 2.2835 | Regression_cos_phi: 0.5262 | Regression_energy: 1.2859 | Regression_eta: 0.5528 | Regression_pt: 1.3118 | Regression_sin_phi: 0.4067 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8464s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0785 | max_loss: 0.0036 | max_backward: 0.3066 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.3809 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6685.8 | uss_MB: 5927.6 | child_rss_MB: 3991.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:18:42,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:19:23,632] INFO : Step 49300/50000 rank0 | Train Loss: 6.9003 | Train Loss Unweighted: 2.4761 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.2115 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1142 | Regression_eta: 0.0107 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7326 | Classification_binary: 2.2115 | Regression_cos_phi: 0.5070 | Regression_energy: 1.2514 | Regression_eta: 0.5256 | Regression_pt: 1.3346 | Regression_sin_phi: 0.3375 | LR: 1.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7752s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1830 | max_loss: 0.0037 | max_backward: 0.2031 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6685.8 | uss_MB: 5927.6 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:19:23,843] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:20:05,391] INFO : Step 49400/50000 rank0 | Train Loss: 7.6710 | Train Loss Unweighted: 2.7216 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4271 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1276 | Regression_eta: 0.0108 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.9005 | Classification_binary: 2.4271 | Regression_cos_phi: 0.5206 | Regression_energy: 1.3979 | Regression_eta: 0.5302 | Regression_pt: 1.4698 | Regression_sin_phi: 0.4248 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2704 | max_optimizer_step: 0.1544 | max_tensorboard_logging: 0.3812 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6685.8 | uss_MB: 5927.6 | child_rss_MB: 3994.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:20:05,608] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:20:45,907] INFO : Step 49500/50000 rank0 | Train Loss: 6.8956 | Train Loss Unweighted: 2.6684 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.4125 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1103 | Regression_eta: 0.0081 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7851 | Classification_binary: 2.4125 | Regression_cos_phi: 0.4595 | Regression_energy: 1.2085 | Regression_eta: 0.3975 | Regression_pt: 1.3059 | Regression_sin_phi: 0.3267 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.0435s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1523 | max_loss: 0.0036 | max_backward: 0.3909 | max_optimizer_step: 0.1568 | max_tensorboard_logging: 0.3955 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6685.8 | uss_MB: 5929.7 | child_rss_MB: 3992.1 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 17:20:46,120] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:21:27,902] INFO : Step 49600/50000 rank0 | Train Loss: 7.4177 | Train Loss Unweighted: 2.6458 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3490 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1327 | Regression_eta: 0.0095 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8389 | Classification_binary: 2.3490 | Regression_cos_phi: 0.4292 | Regression_energy: 1.4537 | Regression_eta: 0.4690 | Regression_pt: 1.4897 | Regression_sin_phi: 0.3881 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8542s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3107 | max_optimizer_step: 0.1523 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6565.4 | uss_MB: 5807.4 | child_rss_MB: 3992.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 17:21:28,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:22:08,664] INFO : Step 49700/50000 rank0 | Train Loss: 7.0558 | Train Loss Unweighted: 2.6374 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3657 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1184 | Regression_eta: 0.0089 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.8113 | Classification_binary: 2.3657 | Regression_cos_phi: 0.3240 | Regression_energy: 1.2973 | Regression_eta: 0.4375 | Regression_pt: 1.3839 | Regression_sin_phi: 0.4361 | LR: 1.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7747s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2478 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.3825 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6566.4 | uss_MB: 5807.9 | child_rss_MB: 3990.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:22:08,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:22:50,571] INFO : Step 49800/50000 rank0 | Train Loss: 7.5939 | Train Loss Unweighted: 2.6136 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.3335 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1196 | Regression_eta: 0.0219 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7707 | Classification_binary: 2.3335 | Regression_cos_phi: 0.4741 | Regression_energy: 1.3103 | Regression_eta: 1.0781 | Regression_pt: 1.3247 | Regression_sin_phi: 0.3026 | LR: 1.00e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8773s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0274 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.3093 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.3987 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6566.4 | uss_MB: 5808.0 | child_rss_MB: 3991.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 17:22:50,786] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:23:32,462] INFO : Step 49900/50000 rank0 | Train Loss: 7.3435 | Train Loss Unweighted: 2.5946 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2906 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1331 | Regression_eta: 0.0117 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7555 | Classification_binary: 2.2906 | Regression_cos_phi: 0.3533 | Regression_energy: 1.4589 | Regression_eta: 0.5766 | Regression_pt: 1.5923 | Regression_sin_phi: 0.3165 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1046 | max_loss: 0.0036 | max_backward: 0.2871 | max_optimizer_step: 0.1502 | max_tensorboard_logging: 0.3011 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6566.4 | uss_MB: 5810.3 | child_rss_MB: 3991.9 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 96.9 +rank0 [2026-10-28 17:23:32,675] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:24:14,060] INFO : Step 50000/50000 rank0 | Train Loss: 7.2474 | Train Loss Unweighted: 2.5476 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.2704 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1197 | Regression_eta: 0.0126 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 46.3956 | Classification_binary: 1.0000 | Regression_cos_phi: 70.4017 | Regression_energy: 10.9570 | Regression_eta: 49.1616 | Regression_pt: 11.9920 | Regression_sin_phi: 63.1786 | Task Weighted Losses: Classification: 0.7453 | Classification_binary: 2.2704 | Regression_cos_phi: 0.4632 | Regression_energy: 1.3118 | Regression_eta: 0.6171 | Regression_pt: 1.3791 | Regression_sin_phi: 0.4605 | LR: 1.00e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8056s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1823 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6566.4 | uss_MB: 5808.3 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 17:24:14,274] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 48.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 91.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:24:14,295] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 17:24:14,474] INFO : Running validation on rank0 +rank0 [2026-10-28 17:24:14,475] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:24:14,475] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:24:15,808] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 17:24:15,956] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:24:16,089] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:24:16,164] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:24:16,233] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:24:16,302] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:24:16,372] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:24:16,442] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:24:16,555] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:24:16,687] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:24:16,820] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:24:16,892] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:24:16,964] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:24:17,032] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:24:17,103] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:24:17,173] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:24:17,273] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:24:17,279] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:24:17,282] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:24:17,285] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:24:17,286] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:24:17,288] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:24:17,289] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:24:17,292] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:24:17,295] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:24:17,298] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:24:17,299] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:24:17,301] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:24:17,304] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:24:17,305] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:24:17,307] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:24:17,310] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:24:25,080] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.2474 | Valid Loss=7.2727 | Train Loss Unweighted=2.5476 | Valid Loss Unweighted=2.6010 | Stale=0 | ETA=0.0m +rank0 [2026-10-28 17:24:25,090] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:24:25,148] INFO : test_dataset: cld_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:24:25,151] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-10-28 17:24:25,338] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:24:27,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:24:28,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:24:28,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:24:29,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:24:29,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:24:29,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:24:29,675] INFO : test_dataset: cld_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:24:29,678] INFO : Running predictions on cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:24:29,808] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:24:31,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:24:32,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:24:33,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:24:34,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:24:34,277] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:24:34,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:24:34,334] INFO : test_dataset: cld_edm_qq_pf, 510 +rank0 [2026-10-28 17:24:34,337] INFO : Running predictions on cld_edm_qq_pf +rank0 [2026-10-28 17:24:34,464] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:24:36,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:24:37,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:24:37,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:24:38,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:24:38,533] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:24:38,541] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:24:38,799] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-10-28 17:24:39,209] INFO : Plotted number of elements +rank0 [2026-10-28 17:24:40,644] INFO : Plotted jets +rank0 [2026-10-28 17:24:41,742] INFO : Plotted jet ratio +rank0 [2026-10-28 17:24:41,774] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:24:42,005] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_ww_fullhad_pf +rank0 [2026-10-28 17:24:42,400] INFO : Plotted number of elements +rank0 [2026-10-28 17:24:43,787] INFO : Plotted jets +rank0 [2026-10-28 17:24:45,530] INFO : Plotted jet ratio +rank0 [2026-10-28 17:24:45,579] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 17:24:45,790] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_pf_heptv2_seed12345_20261028_111202_024020/preds_step_50000/cld_edm_qq_pf +rank0 [2026-10-28 17:24:46,154] INFO : Plotted number of elements +rank0 [2026-10-28 17:24:47,535] INFO : Plotted jets +rank0 [2026-10-28 17:24:48,604] INFO : Plotted jet ratio +rank0 [2026-10-28 17:24:49,126] INFO : Training completed. Total time on device 0: 372.263min diff --git a/cld/clusters/v3.2.0/comparison_metrics/provenance.json b/cld/clusters/v3.2.0/comparison_metrics/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd93dc97277643919f983f16a0633812762afa4 --- /dev/null +++ b/cld/clusters/v3.2.0/comparison_metrics/provenance.json @@ -0,0 +1,24 @@ +{ + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "study": "20261030_clic_cld_checkpoint_summary", + "selected_step": 50000, + "sources": "Original per-run history JSON and train TensorBoard streams, retained alongside these summaries.", + "metrics": { + "f1": "Count-aggregated validation particle F1. Capped matching cost 2.0, acceptance deltaR<0.1 and relative pT error<0.5, no PID requirement.", + "pt_mae": "Mean absolute relative pT error among accepted pairs; conditional and truncated by acceptance, not inclusive resolution.", + "jet_iqr": "Equal mean of ttbar, WW fullhad, qq IQRs of angular-matched predicted/target jet pT ratios.", + "jet_recall": "Equal-process response-qualified jet recall, with abs(pT ratio - 1)<0.5.", + "met": "Full-validation mean absolute error of MET magnitude in GeV.", + "p3": "Detailed 512-entry subset mean norm of event three-momentum difference, GeV.", + "steady_s": "Median seconds/step across adjacent 100-step train TensorBoard intervals, excluding validation boundaries, warmup and session changes.", + "compute_s": "Median instrumented rank-zero forward+loss+backward+optimizer time, seconds.", + "peak_gib": "Maximum recorded allocator peak across ranks and sessions, per logical device. May include prior validation allocations." + }, + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ] +} diff --git a/cld/clusters/v3.2.0/comparison_metrics/study_methodology.md b/cld/clusters/v3.2.0/comparison_metrics/study_methodology.md new file mode 100644 index 0000000000000000000000000000000000000000..d40607cfea1c2d71542e91632ef20b56a9c172a0 --- /dev/null +++ b/cld/clusters/v3.2.0/comparison_metrics/study_methodology.md @@ -0,0 +1,207 @@ +# CLIC and CLD checkpoint summary, 30 October 2026 + +Twenty summary slides compare the eight runs in the 28 October +`clic_cld_backbone_benchmark` campaign. The common endpoint is **50,000 optimizer +steps**. Evolution curves receive priority over endpoint-only comparisons. +This is a snapshot of local results, not a report of current remote job status. + +## Deliverables + +- [PDF slides](output/checkpoint_summary.slides.pdf) +- [Self-contained HTML slides](output/checkpoint_summary.slides.html), arrow-key navigation +- [Source notebook](checkpoint_summary.ipynb) and [executed notebook](output/checkpoint_summary.executed.ipynb) +- [Numerical scorecard](output/scorecard.csv), [summary](output/summary.json), [resource series](output/resources.json) +- `output/slides.json`: generated slide copy, figure paths and source categories +- `output/*.svg` and `output/*.png`: standalone scientific figures + +The figures are generated from archived data by `study_analysis.py`. Tables and +slide copy are HTML text. No external CDN, mutable experiment path, or earlier +study archive is needed to render. This deliverable follows the notebook/HTML/PDF +format in the parent README; it does not include a PowerPoint conversion. + +## Findings + +HEPTv2 improves final jet IQR, MET error and detailed event momentum error in +all four detector/representation pairs. On CLIC hits, F1 rises from 0.7747 to +0.8623 (+8.76 percentage points) and the equal-process jet IQR decreases from +0.1579 to 0.0995 (37.0%). On CLD hits, F1 rises from 0.8458 to 0.8624 +(+1.67 points) and jet IQR decreases from 0.1243 to 0.1040 (16.4%). + +Hit HEPTv2 has higher final particle F1 than track/cluster HEPTv2 in both +detectors. Track/cluster HEPTv2 retains lower conditional particle momentum +error, jet IQR, MET error and event momentum-vector error. These statements +refer to the recorded metrics and configured target populations, not paired +statistical significance or full inclusive momentum resolution. + +The median ordinary training-interval estimates are about 0.11 s/step for PF +attention, 0.40–0.41 s for PF HEPTv2, 2.36–2.38 s for hit attention and +2.84–2.89 s for hit HEPTv2. PF uses one logical accelerator device; hits use +eight. At 50k steps this projects to roughly 55–58 times the device cost for +hit HEPTv2 versus PF HEPTv2, excluding validation, queueing and input +preprocessing. This is not an inference or end-to-end reconstruction benchmark. +Exposed rank-zero batch wait has medians near 2 ms for PF and 17 ms for hits. +No storage byte counters, CPU utilization or bandwidth measurements support a +broader I/O-bottleneck conclusion. + +## Runs and configuration + +All runs use seed 12345, revision +`6b80b983477321cf708e5043fb1d2ec15dd398bc`, global batch 128, 50k scheduled steps, +LAMB, BF16, LR 0.001 with cosine decay, and no compilation. All completed +histories end at 50k. Run IDs record launch preparation on 28 October 2026 at +11:12; hit continuations were submitted on 29 October as recorded in their +manifests. Full log timestamps and exact commands remain archived. + +| Archive name | Exact run ID | Devices | History points | Resume checkpoint | +| --- | --- | ---: | ---: | --- | +| cld_hits_attention | `cld_hits_attention_seed12345_20261028_111201_979187` | 8 | 50 | 35000 | +| cld_hits_heptv2 | `cld_hits_heptv2_seed12345_20261028_111202_875252` | 8 | 49 | 30000 | +| cld_pf_attention | `cld_pf_attention_seed12345_20261028_111201_967206` | 1 | 50 | none | +| cld_pf_heptv2 | `cld_pf_heptv2_seed12345_20261028_111202_024020` | 1 | 50 | none | +| clic_hits_attention | `clic_hits_attention_seed12345_20261028_111202_121851` | 8 | 50 | 35000 | +| clic_hits_heptv2 | `clic_hits_heptv2_seed12345_20261028_111202_012645` | 8 | 50 | 29000 | +| clic_pf_attention | `clic_pf_attention_seed12345_20261028_111202_052283` | 1 | 50 | none | +| clic_pf_heptv2 | `clic_pf_heptv2_seed12345_20261028_111202_849554` | 1 | 50 | none | + +Hardware provenance comes from `scenario-manifest.json`: platform +`lumi_mi250x`, eight logical devices available per platform/node. Actual model +world size is one for track/cluster and eight for hits, verified against resolved +configuration and logs. The generic `gpu_type: l40` in train-config is a stale +recipe default, not evidence that these jobs ran on L40 hardware. Device-hours +count logical MI250X accelerator devices/GCDs, not whole dual-GCD packages. + +All models use width 256, six shared backbone layers. PF attention/HEPTv2 have +3,114,258 / 8,639,954 parameters. Hit attention/HEPTv2 have +7,260,244 / 12,786,964. The hit decoder uses four global-Flash layers and 256 +slots, supervised grouping, tracker-neighborhood features and dedicated tracker +momentum readout. HEPTv2 uses deterministic hashing, block size 128 and activation +checkpointing. These are fixed recipe comparisons, not equal-parameter or +pure-backbone-kernel benchmarks. Losses/objectives and readouts differ between +PF and hit models, so weighted loss is not used as cross-representation evidence. + +Both detectors use ttbar, WW fullhad and qq. Dataset names are +`{clic,cld}_edm_{ttbar,ww_fullhad,qq}_{pf,hits}`, with all ten numbered configs. +PF TFDS is 3.3.0 and hit TFDS is 3.2.2. Exact production/data locations are +preserved only as provenance in original configuration files. Rendering never +reads those locations. At 50k each run has about 2.37 training epochs / 6.4M +training-event presentations, not 6.4M unique events. + +## Metrics and sampling + +Particle curves use history `valid/metrics/particle/*` scalars. The archived +validation matcher minimizes `(deltaR/0.1)^2 + (log(pT ratio)/log(2))^2`, capped +at 2 per pair. Accepted matches require deltaR < 0.1 and relative pT error < 0.5, +without PID agreement. F1/efficiency aggregate counts through the validation +accumulator, rather than averaging process-level F1. Conditional momentum MAE +is mean absolute relative pT error among accepted pairs; the 50% acceptance cut +truncates its tails. It is not a central-68% width or inclusive resolution. + +`nvalid=512` with ten split configs selects 51 entries per config, 510 per +process, 1,530 total. The distributed sampler pads the hit validation collection +to 1,536 entries. Detailed momentum metrics apply to four batches per rank, +512 entries globally for both representations, but rank partitioning changes +which events are in this subset. `momentum/all/event/p3_error_mean` is the mean +norm of the difference between summed predicted and target three-momenta, +not the mean of individual particle momentum errors. Basic MET and energy +metrics use the full validation collection. MET error compares magnitudes. + +Jet curves use top-level `step//jet_ratio/jet_ratio_target_to_pred_pt/*` +scalars from the test/plot cycle. Ratios are predicted / target pT. IQR is +Q75−Q25 among angular-matched jets, with no additional response acceptance cut. +Response-qualified recall counts angular matches satisfying |ratio−1| < 0.5 +relative to all target jets. Each displayed curve is an equal mean of the three +process-specific summaries, not an IQR of pooled jets. `ntest=512` likewise +requests 51 events from each of ten configs per process before distributed +padding; sample identity and duplicate entries were not independently audited. + +PF and hit target means differ slightly (e.g. CLD 70.565 versus 70.655), which +can include both target-definition and sampler effects. We have not verified +exact truth-event identity across representations, nor rescored predictions +with a common event subset. Backbone comparisons within a representation have +the same nominal configuration and validation counts, but have no paired-event +confidence intervals or training-seed replication. Repeated validation/test +inspection makes the comparisons exploratory. Earlier studies have different +data and matching defaults, and their metric values are not mixed into this +campaign. CLD hit HEPTv2 lacks `step_30000.json`; plots connect 29k to 31k without +synthesizing a 30k observation. + +## Resource method + +`extract_resources.py` reads every archived train event file separately with +unbounded scalar retention. It retains the latest wall-time observation for +replayed steps, preserving stream identity to reject cross-session intervals. +All required scalar series have aligned step sets. There are 500 observations +per model at steps 100–50,000 after deduplication. + +Ordinary throughput uses elapsed wall time between adjacent 100-step TensorBoard +observations from the same event file, excluding intervals beginning at a +multiple of 1,000 (which cross validation/test work), intervals before 1,000, +and nonadjacent/restart boundaries. The median seconds per optimizer step is +robust to isolated stalls but is an estimate, not total campaign elapsed time. +Each 100-step interval still contains one diagnostic step. Projected 50k device +cost multiplies this median by 50,000 and by actual world size. It excludes +validation, test plotting, checkpoint cycles at excluded boundaries, scheduling +downtime, startup and abandoned/replayed training work. It does not include PF +tracking/clustering preprocessing. + +Instrumented-step curves and phase breakdowns use rank-zero raw phase timers, +not sums of maxima across different ranks. Compute is forward + loss + backward ++ optimizer. The enclosing `time_model_forward` includes training, backward and +logging, so it is not used as pure forward. `step/time_train_total` includes +synchronized diagnostics and memory logging. Its median can be much greater +than the ordinary interval estimate. The remaining phase is total minus listed +phases; negative residuals are rejected. It includes diagnostic and collective +work not separately plotted. + +Loader wait is the exposed wall time in `next(train_iterator)` on rank zero. +Transfer is the synchronized batch-to-device phase. Neither measures storage +service time, byte throughput, asynchronous worker effort or the slowest rank's +I/O. Resource plots use all raw sampled observations; thick timing lines are +centered 11-sample medians. Input multiplicities quoted on slide 15 are medians +of the per-step, across-rank batch-average element counts. + +Allocator memory is the maximum across ranks of historical peak allocation at +step end, and can include earlier validation allocations. Restart resets it. +This differs from reserved/cached memory and total VRAM occupancy. CPU RSS is +the mean training-process footprint across ranks; it omits child RSS and is not +unique node memory. MB tag values use MiB, divided by 1024 for GiB. The compact +resource JSON also retains child RSS, USS, current reserved memory and padding +for later inspection. No inference benchmark is inferred from these logs. + +**Timer trap:** history `step_valid_time` is calculated as current time minus +session start minus the last training-step duration. It is cumulative and +resets on resume, not a validation-cycle duration. Neither it nor a sum of +history `step_total_time` values is used as validation or total training cost. +The archived revision's `training.py` documents this behavior. + +## Archive and reproduction + +Run from the repository root: + +```bash +bash notebooks/studies/20261030_clic_cld_checkpoint_summary/render.sh +``` + +The script executes the source notebook, which re-extracts resources, regenerates +figures/tables/copy, and emits each slide. It writes the executed notebook and +self-contained HTML, then headless Chrome generates PDF and Poppler renders page +PNGs. Override `PYTHON_BIN` / `CHROME_BIN` if needed. Dependencies are nbformat, +nbclient, IPython, numpy, matplotlib, TensorBoard and a Python 3 Jupyter kernel. +Jupyter needs local socket permissions. Chrome/Chromium is optional for HTML +but required for PDF. `MPLCONFIGDIR` defaults to a temporary writable cache; +scientific inputs and outputs remain study-relative. + +`inputs//` contains complete histories, complete +`plots_step_50000`, the rank-zero `train.log.0` copied as `train.log`, separate +nonempty `tensorboard/train` and `tensorboard/valid` streams, train configuration, +hyperparameters, scenario manifest and resolved particleflow specification. +Files are byte-for-byte copies. `inputs/run_manifest.json` maps descriptive names +to exact original run IDs. `inputs/SHA256SUMS` checks every archived input. +`inputs/sources/` contains the campaign revision's metric, data-sampling and +timing code for definition auditing. No checkpoint, prediction parquet, model +pickle or dataset cache was copied. Inputs total approximately 428.6 MB, mainly +TensorBoard. There is no dependency on a previous study's local artifacts. + +New study files follow the parent directory's local-artifact/Git-ignore policy. +Preserve the full dated folder to retain reproducibility. No private-bucket +upload was requested or performed. diff --git a/cld/clusters/v3.2.0/comparison_metrics/summary.json b/cld/clusters/v3.2.0/comparison_metrics/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3baa4345c2a0014f857f733073c08b7d8fad373d --- /dev/null +++ b/cld/clusters/v3.2.0/comparison_metrics/summary.json @@ -0,0 +1,44 @@ +[ + { + "run": "cld_pf_attention", + "detector": "CLD", + "model": "Track/cluster attention", + "f1": 0.8351587057113647, + "efficiency": 0.8334182500839233, + "pt_mae": 0.08463682979345322, + "jet_iqr": 0.08099334062058232, + "jet_recall": 0.9514983060561701, + "met": 4.993486404418945, + "p3": 13.80346393585205, + "steady_s": 0.11193811893463135, + "devices": 1, + "params": 3114258, + "wait_ms": 2.066373825073242, + "wait_pct": 0.1512281608791759, + "transfer_ms": 1.2848689802922308, + "compute_s": 0.2151729288743809, + "sampled_s": 1.3415029048919678, + "peak_gib": 2.2739763259887695 + }, + { + "run": "cld_pf_heptv2", + "detector": "CLD", + "model": "Track/cluster HEPTv2", + "f1": 0.8469515442848206, + "efficiency": 0.8464965224266052, + "pt_mae": 0.0806594118475914, + "jet_iqr": 0.06921298201372073, + "jet_recall": 0.9531990444995911, + "met": 4.353595733642578, + "p3": 11.977392196655273, + "steady_s": 0.41169955492019655, + "devices": 1, + "params": 8639954, + "wait_ms": 2.012014389038086, + "wait_pct": 0.12240599403264804, + "transfer_ms": 1.2725715059787035, + "compute_s": 0.5150086318608373, + "sampled_s": 1.6560965776443481, + "peak_gib": 3.030461311340332 + } +] diff --git a/cld/clusters/v3.2.0/release.json b/cld/clusters/v3.2.0/release.json new file mode 100644 index 0000000000000000000000000000000000000000..cc1c29cab5ad2d2b59d47dbcd416d3e5db1a648c --- /dev/null +++ b/cld/clusters/v3.2.0/release.json @@ -0,0 +1,72 @@ +{ + "release": "3.2.0", + "detector": "cld", + "representation": "clusters", + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "selected_step": 50000, + "dataset_version": "3.3.0", + "models": { + "attention": { + "run": "cld_pf_attention_seed12345_20261028_111201_967206", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "38f74c605f67cfc429a5394f30856f3d9e65bf40316708ede5eb0aa7597d5000", + "parameters": 3114258, + "global_batch_size": 128, + "accelerator_devices": 1, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8351587057113647, + "metrics/particle/matched/pt_relative_abs_mean": 0.08463682979345322, + "metrics/particle/event/met_abs_error": 4.993486404418945, + "metrics/particle/momentum/all/event/p3_error_mean": 13.80346393585205 + } + }, + "heptv2": { + "run": "cld_pf_heptv2_seed12345_20261028_111202_024020", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "01e13a81252b32cd0ba43135e00c12b229aee6e8a9951af527a956d65e947b2a", + "parameters": 8639954, + "global_batch_size": 128, + "accelerator_devices": 1, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8469515442848206, + "metrics/particle/matched/pt_relative_abs_mean": 0.0806594118475914, + "metrics/particle/event/met_abs_error": 4.353595733642578, + "metrics/particle/momentum/all/event/p3_error_mean": 11.977392196655273 + } + } + }, + "contents": "Final 50k checkpoints, complete available histories, separate train/valid TensorBoard streams, configs, model kwargs, scenario manifests, all training logs, and complete final plots. Prediction parquet files and batch dumps are excluded.", + "checkpoint_selection": "Final scheduled 50k checkpoint; not selected as best by validation or test performance.", + "code_tag": "v3.2.0 is the intended code release label. This upload does not create or verify the Git tag. The training commit above is authoritative.", + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ], + "checkpoint_validation": [ + { + "run": "cld_pf_attention_seed12345_20261028_111201_967206", + "step": 50000, + "parameters": 3114258, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + }, + { + "run": "cld_pf_heptv2_seed12345_20261028_111202_024020", + "step": 50000, + "parameters": 8639954, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + } + ] +}